fetch and build grammars with nix in flake

This commit replaces the out-of-date builder in the flake which relied
on submodules for fetching and the compiler for building. Now we
disable fetching and building explicitly with the environment variable
and then use builtins.fetchGit and a derivation mostly derived from
upstream to compile the grammars.

Anecdotally, this is still quite slow as builtins.fetchGit does not
seem to do shallow clones. I'm not sure I see a way around it though
without recording sha256s, which seems cumbersome.

flake.nix

@@ -33,26 +33,32 @@
         package = "helix";
       };
       overrides = {
-        crateOverrides = common: _: rec {
+        crateOverrides = common: _: {
-          helix-term = prev: {
+          helix-term = prev:
-            # disable fetching and building of tree-sitter grammars in the helix-term build.rs
+            let
-            HELIX_DISABLE_AUTO_GRAMMAR_BUILD = "1";
+              inherit (common) pkgs;
-            buildInputs = (prev.buildInputs or [ ]) ++ [ common.cCompiler.cc.lib ];
+              grammars = pkgs.callPackage ./grammars.nix { };
-            nativeBuildInputs = (prev.nativeBuildInputs or [ ]) ++ [ common.pkgs.makeWrapper ];
+              runtimeDir = pkgs.runCommand "helix-runtime" { } ''
-            preConfigure = ''
+                mkdir -p $out
-              ${prev.preConfigure}
+                ln -s ${common.root}/runtime/* $out
-              rm -r helix-syntax/languages
+                rm -r $out/grammars
-              ln -s ${helix}/helix-syntax/languages helix-syntax/languages
+                ln -s ${grammars} $out/grammars
-              ln -s "$PWD/helix-syntax/languages" languages
+              '';
-              mkdir -p runtime/grammars
+            in
-            '';
+            {
-            postInstall = ''
+              # disable fetching and building of tree-sitter grammars in the helix-term build.rs
-              ${prev.postInstall or ""}
+              HELIX_DISABLE_AUTO_GRAMMAR_BUILD = "1";
-              mkdir -p $out/lib
+              # link languages and theme toml files since helix-term expects them (for tests)
-              cp -r runtime $out/lib
+              preConfigure = "ln -s ${common.root}/{languages.toml,theme.toml,base16_theme.toml} ..";
-              wrapProgram "$out/bin/hx" --set HELIX_RUNTIME "$out/lib/runtime"
+              buildInputs = (prev.buildInputs or [ ]) ++ [ common.cCompiler.cc.lib ];
-            '';
+              nativeBuildInputs = [ pkgs.makeWrapper ];
-          };
+
+              postFixup = ''
+                if [ -f "$out/bin/hx" ]; then
+                  wrapProgram "$out/bin/hx" --set HELIX_RUNTIME "${runtimeDir}"
+                fi
+              '';
+            };
         };
         shell = common: prev: {
           packages = prev.packages ++ (with common.pkgs; [ lld_13 lldb cargo-tarpaulin cargo-flamegraph ]);

grammars.nix

@@ -0,0 +1,87 @@
+{ stdenv, lib, runCommand, yj }:
+let
+  # HACK: nix < 2.6 has a bug in the toml parser, so we convert to JSON
+  # before parsing
+  languages-json = runCommand "languages-toml-to-json" { } ''
+    ${yj}/bin/yj -t < ${./languages.toml} > $out
+  '';
+  languagesConfig =
+    builtins.fromJSON (builtins.readFile (builtins.toPath languages-json));
+  isGitGrammar = (grammar:
+    builtins.hasAttr "source" grammar && builtins.hasAttr "git" grammar.source
+    && builtins.hasAttr "rev" grammar.source);
+  gitGrammars = builtins.filter isGitGrammar languagesConfig.grammar;
+  buildGrammar = grammar:
+    let
+      source = builtins.fetchGit {
+        url = grammar.source.git;
+        rev = grammar.source.rev;
+        allRefs = true;
+      };
+    in stdenv.mkDerivation rec {
+      # see https://github.com/NixOS/nixpkgs/blob/fbdd1a7c0bc29af5325e0d7dd70e804a972eb465/pkgs/development/tools/parsing/tree-sitter/grammar.nix
+
+      pname = "helix-tree-sitter-${grammar.name}";
+      version = grammar.source.rev;
+
+      src = if builtins.hasAttr "subpath" grammar.source then
+        "${source}/${grammar.source.subpath}"
+      else
+        source;
+
+      dontUnpack = true;
+      dontConfigure = true;
+
+      FLAGS = [
+        "-I${src}/src"
+        "-g"
+        "-O3"
+        "-fPIC"
+        "-fno-exceptions"
+        "-Wl,-z,relro,-z,now"
+      ];
+
+      NAME = grammar.name;
+
+      buildPhase = ''
+        runHook preBuild
+
+        if [[ -e "$src/src/scanner.cc" ]]; then
+          $CXX -c "$src/src/scanner.cc" -o scanner.o $FLAGS
+        elif [[ -e "$src/src/scanner.c" ]]; then
+          $CC -c "$src/src/scanner.c" -o scanner.o $FLAGS
+        fi
+
+        $CC -c "$src/src/parser.c" -o parser.o $FLAGS
+        $CXX -shared -o $NAME.so *.o
+
+        ls -al
+
+        runHook postBuild
+      '';
+
+      installPhase = ''
+        runHook preInstall
+        mkdir $out
+        mv $NAME.so $out/
+        runHook postInstall
+      '';
+
+      # Strip failed on darwin: strip: error: symbols referenced by indirect symbol table entries that can't be stripped
+      fixupPhase = lib.optionalString stdenv.isLinux ''
+        runHook preFixup
+        $STRIP $out/$NAME.so
+        runHook postFixup
+      '';
+    };
+  builtGrammars = builtins.map (grammar: {
+    inherit (grammar) name;
+    artifact = buildGrammar grammar;
+  }) gitGrammars;
+  grammarLinks = builtins.map (grammar:
+    "ln -s ${grammar.artifact}/${grammar.name}.so $out/${grammar.name}.so")
+    builtGrammars;
+in runCommand "consolidated-helix-grammars" { } ''
+  mkdir -p $out
+  ${builtins.concatStringsSep "\n" grammarLinks}
+''