implement build_grammars and fetch_grammars

build_grammars adapts the functionality that previously came from
helix-syntax to be used at runtime from the command line flags.

fetch_grammars wraps command-line git to perform the same actions
previously done in the scripts in #1560.

helix-term/src/grammars.rs

@@ -4,39 +4,170 @@ use std::time::SystemTime;
 use std::{
     path::{Path, PathBuf},
     process::Command,
+    sync::mpsc::channel,
 };
 
-use helix_core::syntax::DYLIB_EXTENSION;
+use helix_core::syntax::{GrammarConfiguration, GrammarSource, DYLIB_EXTENSION};
 
 const BUILD_TARGET: &str = env!("BUILD_TARGET");
+const REMOTE_NAME: &str = "origin";
 
-pub fn collect_tree_sitter_dirs(ignore: &[String]) -> Result<Vec<String>> {
+pub fn fetch_grammars() -> Result<()> {
-    let mut dirs = Vec::new();
+    run_parallel(get_grammar_configs(), fetch_grammar, "fetch")
-    let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../helix-syntax/languages");
+}
+
+pub fn build_grammars() -> Result<()> {
+    run_parallel(get_grammar_configs(), build_grammar, "build")
+}
+
+fn run_parallel<F>(grammars: Vec<GrammarConfiguration>, job: F, action: &'static str) -> Result<()>
+where
+    F: Fn(GrammarConfiguration) -> Result<()> + std::marker::Send + 'static + Copy,
+{
+    let mut n_jobs = 0;
+    let pool = threadpool::Builder::new().build();
+    let (tx, rx) = channel();
+
+    for grammar in grammars {
+        let tx = tx.clone();
+        n_jobs += 1;
+
+        pool.execute(move || {
+            let grammar_id = grammar.grammar_id.clone();
+            job(grammar).unwrap_or_else(|err| {
+                eprintln!("Failed to {} grammar '{}'\n{}", action, grammar_id, err)
+            });
+
+            // report progress
+            tx.send(1).unwrap();
+        });
+    }
+    pool.join();
+
+    if rx.try_iter().sum::<usize>() == n_jobs {
+        Ok(())
+    } else {
+        Err(anyhow!("Failed to {} some grammar(s).", action))
+    }
+}
 
-    for entry in fs::read_dir(path)? {
+fn fetch_grammar(grammar: GrammarConfiguration) -> Result<()> {
-        let entry = entry?;
+    if let GrammarSource::Git { remote, revision } = grammar.source {
-        let path = entry.path();
+        let grammar_dir = helix_core::runtime_dir()
+            .join("grammars/sources")
+            .join(grammar.grammar_id.clone());
 
-        if !entry.file_type()?.is_dir() {
+        fs::create_dir_all(grammar_dir.clone()).expect("Could not create grammar directory");
-            continue;
+
+        // create the grammar dir contains a git directory
+        if !grammar_dir.join(".git").is_dir() {
+            git(&grammar_dir, ["init"])?;
+        }
+
+        // ensure the remote matches the configured remote
+        if get_remote_url(&grammar_dir).map_or(true, |s| s.trim_end() != remote) {
+            set_remote(&grammar_dir, &remote)?;
         }
 
-        let dir = path.file_name().unwrap().to_str().unwrap().to_string();
+        // ensure the revision matches the configured revision
+        if get_revision(&grammar_dir).map_or(true, |s| s.trim_end() != revision) {
+            // Fetch the exact revision from the remote.
+            // Supported by server-side git since v2.5.0 (July 2015),
+            // enabled by default on major git hosts.
+            git(&grammar_dir, ["fetch", REMOTE_NAME, &revision])?;
+            git(&grammar_dir, ["checkout", &revision])?;
 
-        // filter ignores
+            println!(
-        if ignore.contains(&dir) {
+                "Grammar '{}' checked out at '{}'.",
-            continue;
+                grammar.grammar_id, revision
+            );
+            Ok(())
+        } else {
+            println!("Grammar '{}' is already up to date.", grammar.grammar_id);
+            Ok(())
         }
-        dirs.push(dir)
+    } else {
+        println!("Skipping local grammar '{}'", grammar.grammar_id);
+        Ok(())
     }
+}
 
-    Ok(dirs)
+// Sets the remote for a repository to the given URL, creating the remote if
+// it does not yet exist.
+fn set_remote(repository: &Path, remote_url: &str) -> Result<String> {
+    git(repository, ["remote", "set-url", REMOTE_NAME, remote_url])
+        .or_else(|_| git(repository, ["remote", "add", REMOTE_NAME, remote_url]))
 }
 
-fn build_library(src_path: &Path, language: &str) -> Result<()> {
+fn get_remote_url(repository: &Path) -> Option<String> {
+    git(repository, ["remote", "get-url", REMOTE_NAME]).ok()
+}
+
+fn get_revision(repository: &Path) -> Option<String> {
+    git(repository, ["rev-parse", "HEAD"]).ok()
+}
+
+// A wrapper around 'git' commands which returns stdout in success and a
+// helpful error message showing the command, stdout, and stderr in error.
+fn git<I, S>(repository: &Path, args: I) -> Result<String>
+where
+    I: IntoIterator<Item = S>,
+    S: AsRef<std::ffi::OsStr>,
+{
+    let output = Command::new("git")
+        .args(args)
+        .current_dir(repository)
+        .output()?;
+
+    if output.status.success() {
+        Ok(String::from_utf8_lossy(&output.stdout).into_owned())
+    } else {
+        // TODO: figure out how to display the git command using `args`
+        Err(anyhow!(
+            "Git command failed.\nStdout: {}\nStderr: {}",
+            String::from_utf8_lossy(&output.stdout),
+            String::from_utf8_lossy(&output.stderr),
+        ))
+    }
+}
+
+fn build_grammar(grammar: GrammarConfiguration) -> Result<()> {
+    let grammar_dir = if let GrammarSource::Local { ref path } = grammar.source {
+        PathBuf::from(path)
+    } else {
+        helix_core::runtime_dir()
+            .join("grammars/sources")
+            .join(grammar.grammar_id.clone())
+    };
+
+    grammar_dir.read_dir().with_context(|| {
+        format!(
+            "The directory {:?} is empty, you probably need to use 'hx --fetch-grammars'?",
+            grammar_dir
+        )
+    })?;
+
+    let path = match grammar.path {
+        Some(ref subpath) => grammar_dir.join(subpath),
+        None => grammar_dir,
+    }
+    .join("src");
+
+    build_tree_sitter_library(&path, grammar)
+}
+
+// Returns the set of grammar configurations the user requests.
+// Grammars are configured in the default and user `languages.toml` and are
+// merged. The `grammar_selection` key of the config is then used to filter
+// down all grammars into a subset of the user's choosing.
+fn get_grammar_configs() -> Vec<GrammarConfiguration> {
+    let config = helix_core::config::user_syntax_loader().expect("Could not parse languages.toml");
+
+    config.grammar
+}
+
+fn build_tree_sitter_library(src_path: &Path, grammar: GrammarConfiguration) -> Result<()> {
     let header_path = src_path;
-    // let grammar_path = src_path.join("grammar.json");
     let parser_path = src_path.join("parser.c");
     let mut scanner_path = src_path.join("scanner.c");
 
@@ -50,16 +181,20 @@ fn build_library(src_path: &Path, language: &str) -> Result<()> {
             None
         }
     };
-    let parser_lib_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../runtime/grammars");
+    let parser_lib_path = helix_core::runtime_dir().join("../runtime/grammars");
-    let mut library_path = parser_lib_path.join(language);
+    let mut library_path = parser_lib_path.join(grammar.grammar_id.clone());
     library_path.set_extension(DYLIB_EXTENSION);
 
     let recompile = needs_recompile(&library_path, &parser_path, &scanner_path)
-        .with_context(|| "Failed to compare source and binary timestamps")?;
+        .context("Failed to compare source and binary timestamps")?;
 
     if !recompile {
+        println!("Grammar '{}' is already built.", grammar.grammar_id);
         return Ok(());
     }
+
+    println!("Building grammar '{}'", grammar.grammar_id);
+
     let mut config = cc::Build::new();
     config
         .cpp(true)
@@ -112,9 +247,7 @@ fn build_library(src_path: &Path, language: &str) -> Result<()> {
         }
     }
 
-    let output = command
+    let output = command.output().context("Failed to execute C compiler")?;
-        .output()
-        .with_context(|| "Failed to execute C compiler")?;
     if !output.status.success() {
         return Err(anyhow!(
             "Parser compilation failed.\nStdout: {}\nStderr: {}",
@@ -125,6 +258,7 @@ fn build_library(src_path: &Path, language: &str) -> Result<()> {
 
     Ok(())
 }
+
 fn needs_recompile(
     lib_path: &Path,
     parser_c_path: &Path,
@@ -148,28 +282,3 @@ fn needs_recompile(
 fn mtime(path: &Path) -> Result<SystemTime> {
     Ok(fs::metadata(path)?.modified()?)
 }
-
-pub fn build_dir(dir: &str, language: &str) {
-    println!("Build language {}", language);
-    if PathBuf::from(env!("CARGO_MANIFEST_DIR"))
-        .join("../helix-syntax/languages")
-        .join(dir)
-        .read_dir()
-        .unwrap()
-        .next()
-        .is_none()
-    {
-        eprintln!(
-            "The directory {} is empty, you probably need to use './scripts/grammars sync'?",
-            dir
-        );
-        std::process::exit(1);
-    }
-
-    let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
-        .join("../helix-syntax/languages")
-        .join(dir)
-        .join("src");
-
-    build_library(&path, language).unwrap();
-}

helix-term/src/main.rs

@@ -104,6 +104,16 @@ FLAGS:
         std::process::exit(0);
     }
 
+    if args.fetch_grammars {
+        helix_term::grammars::fetch_grammars()?;
+        return Ok(0);
+    }
+
+    if args.build_grammars {
+        helix_term::grammars::build_grammars()?;
+        return Ok(0);
+    }
+
     let conf_dir = helix_core::config_dir();
     if !conf_dir.exists() {
         std::fs::create_dir_all(&conf_dir).ok();