diff --git a/Cargo.lock b/Cargo.lock index 21fd2010fa3..04be6b1aab6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6269,6 +6269,7 @@ dependencies = [ "env_logger 0.11.8", "extension", "fs", + "futures 0.3.32", "gpui_platform", "language", "log", diff --git a/Cargo.toml b/Cargo.toml index 037afc59bfe..6e88facf82b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -591,6 +591,7 @@ fork = "0.4.0" futures = "0.3.32" futures-concurrency = "7.7.1" futures-lite = "1.13" +futures-util = "0.3.32" gh-workflow = { git = "https://github.com/zed-industries/gh-workflow", rev = "37f3c0575d379c218a9c455ee67585184e40d43f" } globset = "0.4" diff --git a/crates/extension/src/extension_builder.rs b/crates/extension/src/extension_builder.rs index 2fc50434603..6229da6b8dc 100644 --- a/crates/extension/src/extension_builder.rs +++ b/crates/extension/src/extension_builder.rs @@ -1,21 +1,26 @@ use crate::{ - ExtensionLibraryKind, ExtensionManifest, GrammarManifestEntry, build_debug_adapter_schema_path, - parse_wasm_extension_version, + ExtensionLibraryKind, ExtensionManifest, GrammarManifestEntry, parse_wasm_extension_version, }; use ::fs::Fs; use anyhow::{Context as _, Result, bail}; -use futures::{StreamExt, io}; +use futures::{ + FutureExt, StreamExt, + channel::oneshot::{self, Sender}, + io, +}; use heck::ToSnakeCase; use http_client::{self, AsyncBody, HttpClient}; use language::LanguageConfig; +use semver::Version; use serde::Deserialize; use std::{ env, fs, mem, + num::NonZeroUsize, + ops::Not, path::{Path, PathBuf}, - str::FromStr, sync::Arc, }; -use util::{command::Stdio, rel_path::PathExt}; +use util::{ResultExt, command::Stdio, rel_path::PathExt}; use wasm_encoder::{ComponentSectionId, Encode as _, RawSection, Section as _}; use wasmparser::Parser; @@ -44,13 +49,24 @@ pub struct ExtensionBuilder { pub http: Arc, } +pub enum CompilationConcurrency { + Unbounded, + Bounded(NonZeroUsize), +} + +const DEFAULT_COMPILATION_CONCURRENCY: NonZeroUsize = NonZeroUsize::new(3).unwrap(); + pub struct CompileExtensionOptions { pub release: bool, + pub max_concurrency: CompilationConcurrency, } impl CompileExtensionOptions { pub const fn dev() -> Self { - Self { release: false } + Self { + release: false, + max_concurrency: CompilationConcurrency::Bounded(DEFAULT_COMPILATION_CONCURRENCY), + } } } @@ -79,7 +95,9 @@ impl ExtensionBuilder { options: CompileExtensionOptions, fs: Arc, ) -> Result<()> { - populate_defaults(extension_manifest, extension_dir, fs).await?; + let start = std::time::Instant::now(); + + populate_defaults(extension_manifest, extension_dir, fs.clone()).await?; if extension_dir.is_relative() { bail!( @@ -88,58 +106,116 @@ impl ExtensionBuilder { ); } - fs::create_dir_all(&self.cache_dir).context("failed to create cache dir")?; + fs.create_dir(&self.cache_dir) + .await + .context("failed to create cache dir")?; - if extension_manifest.lib.kind == Some(ExtensionLibraryKind::Rust) { - log::info!("compiling Rust extension {}", extension_dir.display()); - self.compile_rust_extension(extension_dir, extension_manifest, options) - .await - .context("failed to compile Rust extension")?; - log::info!("compiled Rust extension {}", extension_dir.display()); - } + let (tx, mut rx) = oneshot::channel(); - for (debug_adapter_name, meta) in &mut extension_manifest.debug_adapters { - let debug_adapter_schema_path = - extension_dir.join(build_debug_adapter_schema_path(debug_adapter_name, meta)?); + let clang_path = extension_manifest.grammars.is_empty().not().then(|| { + std::iter::repeat_n( + async { + self.install_wasi_sdk_if_needed() + .await + .log_err() + .map(Arc::new) + } + .shared(), + extension_manifest.grammars.len(), + ) + }); - let debug_adapter_schema = fs::read_to_string(&debug_adapter_schema_path) - .with_context(|| { - format!("failed to read debug adapter schema for `{debug_adapter_name}` from `{debug_adapter_schema_path:?}`") - })?; - _ = serde_json::Value::from_str(&debug_adapter_schema).with_context(|| { - format!("Debug adapter schema for `{debug_adapter_name}` (path: `{debug_adapter_schema_path:?}`) is not a valid JSON") - })?; - } - for (grammar_name, grammar_metadata) in &extension_manifest.grammars { - let snake_cased_grammar_name = grammar_name.to_snake_case(); - if grammar_name.as_ref() != snake_cased_grammar_name.as_str() { - bail!( - "grammar name '{grammar_name}' must be written in snake_case: {snake_cased_grammar_name}" - ); + let rust_compilation_task = + (extension_manifest.lib.kind == Some(ExtensionLibraryKind::Rust)).then(|| { + async { + log::info!("compiling Rust extension {}", extension_dir.display()); + self.compile_rust_extension(extension_dir, extension_manifest, tx, &options) + .await + .context("failed to compile Rust extension")?; + + log::info!("compiled Rust extension {}", extension_dir.display()); + Ok(()) + } + .boxed() + }); + + let grammar_compilation_tasks = extension_manifest + .grammars + .iter() + .zip(clang_path.into_iter().flatten()) + .map(|((grammar_name, grammar_metadata), clang_path_task)| { + async move { + let snake_cased_grammar_name = grammar_name.to_snake_case(); + if grammar_name.as_ref() != snake_cased_grammar_name.as_str() { + bail!( + "grammar name '{grammar_name}' must be \ + written in snake_case: {snake_cased_grammar_name}" + ); + } + + log::info!( + "compiling grammar {grammar_name} for extension {}", + extension_dir.display() + ); + + let clang_path = clang_path_task + .await + .context("Failed to resolve clang path")?; + + self.compile_grammar( + extension_dir, + grammar_name.as_ref(), + grammar_metadata, + &clang_path, + ) + .await + .with_context(|| format!("failed to compile grammar '{grammar_name}'"))?; + log::info!( + "compiled grammar {grammar_name} for extension {}", + extension_dir.display() + ); + + Ok(()) + } + .boxed() + }); + + let tasks = rust_compilation_task + .into_iter() + .chain(grammar_compilation_tasks) + .collect::>(); + + match options.max_concurrency { + CompilationConcurrency::Unbounded => { + futures::future::try_join_all(tasks).await?; } + CompilationConcurrency::Bounded(max_concurrency) => { + let mut stream = futures::stream::iter(tasks).buffered(max_concurrency.get()); - log::info!( - "compiling grammar {grammar_name} for extension {}", - extension_dir.display() - ); - self.compile_grammar(extension_dir, grammar_name.as_ref(), grammar_metadata) - .await - .with_context(|| format!("failed to compile grammar '{grammar_name}'"))?; - log::info!( - "compiled grammar {grammar_name} for extension {}", - extension_dir.display() - ); + while let Some(result) = stream.next().await { + result?; + } + } } - log::info!("finished compiling extension {}", extension_dir.display()); + if let Ok(version) = rx.try_recv() { + extension_manifest.lib.version = version; + } + + log::info!( + "finished compiling extension {} in {time:.2}s", + extension_dir.display(), + time = start.elapsed().as_secs_f64(), + ); Ok(()) } async fn compile_rust_extension( &self, extension_dir: &Path, - manifest: &mut ExtensionManifest, - options: CompileExtensionOptions, + manifest: &ExtensionManifest, + wasm_extension_api_version_tx: Sender, + options: &CompileExtensionOptions, ) -> anyhow::Result<()> { self.install_rust_wasm_target_if_needed().await?; @@ -201,7 +277,9 @@ impl ExtensionBuilder { let wasm_extension_api_version = parse_wasm_extension_version(&manifest.id, &component_bytes) .context("compiled wasm did not contain a valid zed extension api version")?; - manifest.lib.version = Some(wasm_extension_api_version); + wasm_extension_api_version_tx + .send(wasm_extension_api_version) + .map_err(|_| anyhow::anyhow!("Failed to send API version"))?; let extension_file = extension_dir.join("extension.wasm"); fs::write(extension_file.clone(), &component_bytes) @@ -221,9 +299,8 @@ impl ExtensionBuilder { extension_dir: &Path, grammar_name: &str, grammar_metadata: &GrammarManifestEntry, + clang_path: &Path, ) -> Result<()> { - let clang_path = self.install_wasi_sdk_if_needed().await?; - let mut grammar_repo_dir = extension_dir.to_path_buf(); grammar_repo_dir.extend(["grammars", grammar_name]); diff --git a/crates/extension_cli/Cargo.toml b/crates/extension_cli/Cargo.toml index 2170647845b..623187faaa3 100644 --- a/crates/extension_cli/Cargo.toml +++ b/crates/extension_cli/Cargo.toml @@ -18,6 +18,7 @@ clap = { workspace = true, features = ["derive"] } cloud_api_types.workspace = true env_logger.workspace = true extension.workspace = true +futures.workspace = true fs.workspace = true gpui_platform.workspace = true language.workspace = true diff --git a/crates/extension_cli/src/main.rs b/crates/extension_cli/src/main.rs index 0b7c491ec11..921990d284c 100644 --- a/crates/extension_cli/src/main.rs +++ b/crates/extension_cli/src/main.rs @@ -3,12 +3,15 @@ use std::collections::HashMap; use std::env; use std::fs; use std::path::{Path, PathBuf}; +use std::str::FromStr as _; use std::sync::Arc; -use ::fs::{CopyOptions, Fs, RealFs, copy_recursive}; +use ::fs::{CopyOptions, Fs, RealFs, RemoveOptions, copy_recursive}; use anyhow::{Context as _, Result, anyhow, bail}; use clap::Parser; use cloud_api_types::ExtensionProvides; +use extension::build_debug_adapter_schema_path; +use extension::extension_builder::CompilationConcurrency; use extension::extension_builder::{CompileExtensionOptions, ExtensionBuilder}; use extension::{ExtensionManifest, ExtensionSnippets}; use language::LanguageConfig; @@ -47,6 +50,11 @@ async fn main() -> Result<()> { .source_dir .canonicalize() .context("failed to canonicalize source_dir")?; + + fs.create_dir(&args.scratch_dir) + .await + .context("failed to create scratch dir")?; + let scratch_dir = args .scratch_dir .canonicalize() @@ -75,7 +83,10 @@ async fn main() -> Result<()> { .compile_extension( &extension_path, &mut manifest, - CompileExtensionOptions { release: true }, + CompileExtensionOptions { + release: true, + max_concurrency: CompilationConcurrency::Unbounded, + }, fs.clone(), ) .await @@ -88,9 +99,18 @@ async fn main() -> Result<()> { test_languages(&manifest, &extension_path, &grammars)?; test_themes(&manifest, &extension_path, fs.clone()).await?; test_snippets(&manifest, &extension_path, fs.clone()).await?; + test_debug_adapter_schemas(&manifest, &extension_path, fs.clone()).await?; let archive_dir = output_dir.join("archive"); - fs::remove_dir_all(&archive_dir).ok(); + fs.remove_dir( + &archive_dir, + RemoveOptions { + recursive: true, + ignore_if_not_exists: true, + }, + ) + .await + .ok(); copy_extension_resources(&manifest, &extension_path, &archive_dir, fs.clone()) .await .context("failed to copy extension resources")?; @@ -120,8 +140,16 @@ async fn main() -> Result<()> { wasm_api_version: manifest.lib.version.map(|version| version.to_string()), provides: extension_provides, })?; - fs::remove_dir_all(&archive_dir)?; - fs::write(output_dir.join("manifest.json"), manifest_json.as_bytes())?; + fs.remove_dir( + &archive_dir, + RemoveOptions { + recursive: true, + ignore_if_not_exists: false, + }, + ) + .await?; + fs.write(&output_dir.join("manifest.json"), manifest_json.as_bytes()) + .await?; Ok(()) } @@ -132,68 +160,107 @@ async fn copy_extension_resources( output_dir: &Path, fs: Arc, ) -> Result<()> { - fs::create_dir_all(output_dir).context("failed to create output dir")?; + fs.create_dir(output_dir) + .await + .context("failed to create output dir")?; let manifest_toml = toml::to_string(&manifest).context("failed to serialize manifest")?; - fs::write(output_dir.join("extension.toml"), &manifest_toml) + fs.write(&output_dir.join("extension.toml"), manifest_toml.as_bytes()) + .await .context("failed to write extension.toml")?; if manifest.lib.kind.is_some() { - fs::copy( - extension_path.join("extension.wasm"), - output_dir.join("extension.wasm"), + fs.copy_file( + &extension_path.join("extension.wasm"), + &output_dir.join("extension.wasm"), + CopyOptions { + overwrite: true, + ignore_if_exists: false, + }, ) + .await .context("failed to copy extension.wasm")?; } if !manifest.grammars.is_empty() { let source_grammars_dir = extension_path.join("grammars"); let output_grammars_dir = output_dir.join("grammars"); - fs::create_dir_all(&output_grammars_dir)?; - for grammar_name in manifest.grammars.keys() { - let mut grammar_filename = PathBuf::from(grammar_name.as_ref()); - grammar_filename.set_extension("wasm"); - fs::copy( - source_grammars_dir.join(&grammar_filename), - output_grammars_dir.join(&grammar_filename), - ) - .with_context(|| format!("failed to copy grammar '{}'", grammar_filename.display()))?; - } + fs.create_dir(&output_grammars_dir).await?; + futures::future::try_join_all(manifest.grammars.keys().map(|grammar_name| { + let fs = fs.clone(); + let source_grammars_dir = source_grammars_dir.as_path(); + let output_grammars_dir = output_grammars_dir.as_path(); + async move { + let mut grammar_filename = PathBuf::from(grammar_name.as_ref()); + grammar_filename.set_extension("wasm"); + fs.copy_file( + &source_grammars_dir.join(&grammar_filename), + &output_grammars_dir.join(&grammar_filename), + CopyOptions { + overwrite: true, + ignore_if_exists: false, + }, + ) + .await + .with_context(|| format!("failed to copy grammar '{}'", grammar_filename.display())) + } + })) + .await?; } if !manifest.themes.is_empty() { let output_themes_dir = output_dir.join("themes"); - fs::create_dir_all(&output_themes_dir)?; - for theme_path in &manifest.themes { - let theme_path = theme_path.as_std_path(); - fs::copy( - extension_path.join(theme_path), - output_themes_dir.join(theme_path.file_name().context("invalid theme path")?), - ) - .with_context(|| format!("failed to copy theme '{}'", theme_path.display()))?; - } + fs.create_dir(&output_themes_dir).await?; + futures::future::try_join_all(manifest.themes.iter().map(|theme_path| { + let fs = fs.clone(); + let output_themes_dir = output_themes_dir.as_path(); + async move { + let theme_path = theme_path.as_std_path(); + fs.copy_file( + &extension_path.join(theme_path), + &output_themes_dir.join(theme_path.file_name().context("invalid theme path")?), + CopyOptions { + overwrite: true, + ignore_if_exists: false, + }, + ) + .await + .with_context(|| format!("failed to copy theme '{}'", theme_path.display())) + } + })) + .await?; } if !manifest.icon_themes.is_empty() { let output_icon_themes_dir = output_dir.join("icon_themes"); - fs::create_dir_all(&output_icon_themes_dir)?; - for icon_theme_path in &manifest.icon_themes { - let icon_theme_path = icon_theme_path.as_std_path(); - fs::copy( - extension_path.join(icon_theme_path), - output_icon_themes_dir.join( - icon_theme_path - .file_name() - .context("invalid icon theme path")?, - ), - ) - .with_context(|| { - format!("failed to copy icon theme '{}'", icon_theme_path.display()) - })?; - } + fs.create_dir(&output_icon_themes_dir).await?; + futures::future::try_join_all(manifest.icon_themes.iter().map(|icon_theme_path| { + let fs = fs.clone(); + let output_icon_themes_dir = output_icon_themes_dir.as_path(); + async move { + let icon_theme_path = icon_theme_path.as_std_path(); + fs.copy_file( + &extension_path.join(icon_theme_path), + &output_icon_themes_dir.join( + icon_theme_path + .file_name() + .context("invalid icon theme path")?, + ), + CopyOptions { + overwrite: true, + ignore_if_exists: false, + }, + ) + .await + .with_context(|| { + format!("failed to copy icon theme '{}'", icon_theme_path.display()) + }) + } + })) + .await?; let output_icons_dir = output_dir.join("icons"); - fs::create_dir_all(&output_icons_dir)?; + fs.create_dir(&output_icons_dir).await?; copy_recursive( fs.as_ref(), &extension_path.join("icons"), @@ -209,73 +276,90 @@ async fn copy_extension_resources( if !manifest.languages.is_empty() { let output_languages_dir = output_dir.join("languages"); - fs::create_dir_all(&output_languages_dir)?; - for language_path in &manifest.languages { - let language_path = language_path.as_std_path(); - copy_recursive( - fs.as_ref(), - &extension_path.join(language_path), - &output_languages_dir - .join(language_path.file_name().context("invalid language path")?), - CopyOptions { - overwrite: true, - ignore_if_exists: false, - }, - ) - .await - .with_context(|| { - format!("failed to copy language dir '{}'", language_path.display()) - })?; - } + fs.create_dir(&output_languages_dir).await?; + futures::future::try_join_all(manifest.languages.iter().map(|language_path| { + let fs = fs.clone(); + let output_languages_dir = output_languages_dir.clone(); + async move { + let language_path = language_path.as_std_path(); + copy_recursive( + fs.as_ref(), + &extension_path.join(language_path), + &output_languages_dir + .join(language_path.file_name().context("invalid language path")?), + CopyOptions { + overwrite: true, + ignore_if_exists: false, + }, + ) + .await + .with_context(|| { + format!("failed to copy language dir '{}'", language_path.display()) + }) + } + })) + .await?; } if !manifest.debug_adapters.is_empty() { - for (debug_adapter, entry) in &manifest.debug_adapters { - let schema_path = extension::build_debug_adapter_schema_path(debug_adapter, entry)?; - let parent = schema_path - .parent() - .with_context(|| format!("invalid empty schema path for {debug_adapter}"))?; - let schema_path = schema_path.as_std_path(); - fs::create_dir_all(output_dir.join(parent))?; - copy_recursive( - fs.as_ref(), - &extension_path.join(&schema_path), - &output_dir.join(&schema_path), - CopyOptions { - overwrite: true, - ignore_if_exists: false, - }, - ) - .await - .with_context(|| { - format!( - "failed to copy debug adapter schema '{}'", - schema_path.display(), - ) - })?; - } + futures::future::try_join_all(manifest.debug_adapters.iter().map( + |(debug_adapter, entry)| { + let fs = fs.clone(); + let debug_adapter = debug_adapter.clone(); + async move { + let schema_path = + extension::build_debug_adapter_schema_path(&debug_adapter, &entry)?; + let parent = schema_path.parent().with_context(|| { + format!("invalid empty schema path for {debug_adapter}") + })?; + let schema_path = schema_path.as_std_path(); + fs.create_dir(&output_dir.join(parent)).await?; + copy_recursive( + fs.as_ref(), + &extension_path.join(schema_path), + &output_dir.join(schema_path), + CopyOptions { + overwrite: true, + ignore_if_exists: false, + }, + ) + .await + .with_context(|| { + format!( + "failed to copy debug adapter schema '{}'", + schema_path.display(), + ) + }) + } + }, + )) + .await?; } if let Some(snippets) = manifest.snippets.as_ref() { - for snippets_path in snippets.paths() { - let parent = snippets_path.parent(); - if let Some(parent) = parent.filter(|p| p.components().next().is_some()) { - fs::create_dir_all(output_dir.join(parent))?; + futures::future::try_join_all(snippets.paths().map(|snippets_path| { + let fs = fs.clone(); + async move { + let parent = snippets_path.parent(); + if let Some(parent) = parent.filter(|p| p.components().next().is_some()) { + fs.create_dir(&output_dir.join(parent)).await?; + } + copy_recursive( + fs.as_ref(), + &extension_path.join(&snippets_path), + &output_dir.join(&snippets_path), + CopyOptions { + overwrite: true, + ignore_if_exists: false, + }, + ) + .await + .with_context(|| { + format!("failed to copy snippets from '{}'", snippets_path.display()) + }) } - copy_recursive( - fs.as_ref(), - &extension_path.join(&snippets_path), - &output_dir.join(&snippets_path), - CopyOptions { - overwrite: true, - ignore_if_exists: false, - }, - ) - .await - .with_context(|| { - format!("failed to copy snippets from '{}'", snippets_path.display()) - })?; - } + })) + .await?; } Ok(()) @@ -486,6 +570,40 @@ async fn test_snippets( Ok(()) } +async fn test_debug_adapter_schemas( + manifest: &ExtensionManifest, + extension_path: &Path, + fs: Arc, +) -> Result<()> { + futures::future::try_join_all(manifest.debug_adapters.iter().map( + |(debug_adapter_name, meta)| { + let fs = fs.clone(); + async move { + let debug_adapter_schema_path = + extension_path.join(build_debug_adapter_schema_path(debug_adapter_name, meta)?); + + let debug_adapter_schema = + fs.load(&debug_adapter_schema_path).await.with_context(|| { + anyhow::anyhow!( + "failed to read debug adapter schema for \ + `{debug_adapter_name}` from `{debug_adapter_schema_path:?}`" + ) + })?; + _ = serde_json::Value::from_str(&debug_adapter_schema).with_context(|| { + anyhow::anyhow!( + "Debug adapter schema for `{debug_adapter_name}`\ + (path: `{debug_adapter_schema_path:?}`) is not a valid JSON" + ) + })?; + + Ok(()) + } + }, + )) + .await + .map(|_| ()) +} + #[cfg(test)] mod tests { use cloud_api_types::ExtensionProvides; diff --git a/crates/extension_host/benches/extension_compilation_benchmark.rs b/crates/extension_host/benches/extension_compilation_benchmark.rs index b6dade97183..e8abdffbd86 100644 --- a/crates/extension_host/benches/extension_compilation_benchmark.rs +++ b/crates/extension_host/benches/extension_compilation_benchmark.rs @@ -4,7 +4,7 @@ use criterion::{BatchSize, BenchmarkId, Criterion, criterion_group, criterion_ma use extension::{ ExtensionCapability, ExtensionHostProxy, ExtensionLibraryKind, ExtensionManifest, LanguageServerManifestEntry, LibManifestEntry, SchemaVersion, - extension_builder::{CompileExtensionOptions, ExtensionBuilder}, + extension_builder::{CompilationConcurrency, CompileExtensionOptions, ExtensionBuilder}, }; use extension_host::wasm_host::WasmHost; use fs::{Fs, RealFs}; @@ -76,7 +76,10 @@ fn wasm_bytes(cx: &TestAppContext, manifest: &mut ExtensionManifest, fs: Arc