Merge pull request #82155 from kolloch/buildRustCrate_findMatchingDir

buildRustCrate: Search for matching Cargo.toml in sub directories

pkgs/build-support/rust/build-rust-crate/build-crate.nix

@@ -1,4 +1,4 @@
-{ lib, stdenv, echo_build_heading, noisily, mkRustcDepArgs, rust }:
+{ lib, stdenv, mkRustcDepArgs, rust }:
 { crateName,
   dependencies,
   crateFeatures, crateRenames, libName, release, libPath,
@@ -35,16 +35,13 @@
     build_bin = if buildTests then "build_bin_test" else "build_bin";
   in ''
     runHook preBuild
-    ${echo_build_heading colors}
-    ${noisily colors verbose}
-
+ 
     # configure & source common build functions
     LIB_RUSTC_OPTS="${libRustcOpts}"
     BIN_RUSTC_OPTS="${binRustcOpts}"
     LIB_EXT="${stdenv.hostPlatform.extensions.sharedLibrary}"
     LIB_PATH="${libPath}"
     LIB_NAME="${libName}"
-    source ${./lib.sh}
 
     CRATE_NAME='${lib.replaceStrings ["-"] ["_"] libName}'
 

pkgs/build-support/rust/build-rust-crate/configure-crate.nix

@@ -1,4 +1,4 @@
-{ lib, stdenv, echo_build_heading, noisily, mkRustcDepArgs }:
+{ lib, stdenv, echo_colored, noisily, mkRustcDepArgs }:
 { build
 , buildDependencies
 , colors
@@ -31,10 +31,25 @@ let version_ = lib.splitString "-" crateVersion;
     completeDepsDir = lib.concatStringsSep " " completeDeps;
     completeBuildDepsDir = lib.concatStringsSep " " completeBuildDeps;
 in ''
-  cd ${workspace_member}
-  runHook preConfigure
-  ${echo_build_heading colors}
+  ${echo_colored colors}
   ${noisily colors verbose}
+  source ${./lib.sh}
+
+  ${lib.optionalString (workspace_member != null) ''
+  noisily cd "${workspace_member}"
+''}
+  ${lib.optionalString (workspace_member == null) ''
+  echo_colored "Searching for matching Cargo.toml (${crateName})" 
+  local cargo_toml_dir=$(matching_cargo_toml_dir "${crateName}")
+  if [ -z "$cargo_toml_dir" ]; then
+    echo_error "ERROR configuring ${crateName}: No matching Cargo.toml in $(pwd) found." >&2
+    exit 23
+  fi
+  noisily cd "$cargo_toml_dir"
+''}
+
+  runHook preConfigure
+ 
   symlink_dependency() {
     # $1 is the nix-store path of a dependency
     # $2 is the target path

pkgs/build-support/rust/build-rust-crate/default.nix

@@ -4,7 +4,7 @@
 # This can be useful for deploying packages with NixOps, and to share
 # binary dependencies between projects.
 
-{ lib, stdenv, defaultCrateOverrides, fetchCrate, rustc, rust }:
+{ lib, stdenv, defaultCrateOverrides, fetchCrate, rustc, rust, cargo, jq }:
 
 let
     # This doesn't appear to be officially documented anywhere yet.
@@ -29,14 +29,14 @@ let
            " --extern ${name}=${dep.lib}/lib/lib${extern}-${dep.metadata}${stdenv.hostPlatform.extensions.sharedLibrary}")
       ) dependencies;
 
-   inherit (import ./log.nix { inherit lib; }) noisily echo_build_heading;
+   inherit (import ./log.nix { inherit lib; }) noisily echo_colored;
 
    configureCrate = import ./configure-crate.nix {
-     inherit lib stdenv echo_build_heading noisily mkRustcDepArgs;
+     inherit lib stdenv echo_colored noisily mkRustcDepArgs;
    };
 
    buildCrate = import ./build-crate.nix {
-     inherit lib stdenv echo_build_heading noisily mkRustcDepArgs rust;
+     inherit lib stdenv mkRustcDepArgs rust;
    };
 
    installCrate = import ./install-crate.nix { inherit stdenv; };
@@ -88,7 +88,7 @@ stdenv.mkDerivation (rec {
 
     src = crate.src or (fetchCrate { inherit (crate) crateName version sha256; });
     name = "rust_${crate.crateName}-${crate.version}${lib.optionalString buildTests_ "-test"}";
-    depsBuildBuild = [ rust stdenv.cc ];
+    depsBuildBuild = [ rust stdenv.cc cargo jq ];
     buildInputs = (crate.buildInputs or []) ++ buildInputs_;
     dependencies = map lib.getLib dependencies_;
     buildDependencies = map lib.getLib buildDependencies_;
@@ -114,6 +114,8 @@ stdenv.mkDerivation (rec {
       in lib.substring 0 10 hashedMetadata;
 
     build = crate.build or "";
+    # Either set to a concrete sub path to the crate root
+    # or use `null` for auto-detect.
     workspace_member = crate.workspace_member or ".";
     crateVersion = crate.version;
     crateDescription = crate.description or "";

pkgs/build-support/rust/build-rust-crate/lib.sh

@@ -1,3 +1,11 @@
+echo_build_heading() {
+  if (( $# == 1 )); then
+    echo_colored "Building $1"
+  else
+    echo_colored "Building $1 ($2)"
+  fi
+}
+
 build_lib() {
   lib_src=$1
   echo_build_heading $lib_src ${libName}
@@ -132,7 +140,41 @@ search_for_bin_path() {
   done
 
   if [[ -z "$BIN_PATH" ]]; then
-    echo "failed to find file for binary target: $BIN_NAME" >&2
+    echo_error "ERROR: failed to find file for binary target: $BIN_NAME" >&2
     exit 1
   fi
 }
+
+# Extracts cargo_toml_path of the matching crate.
+matching_cargo_toml_path() {
+  local manifest_path="$1"
+  local expected_crate_name="$2"
+
+  # If the Cargo.toml is not a workspace root,
+  # it will only contain one package in ".packages"
+  # because "--no-deps" suppressed dependency resolution.
+  #
+  # But to make it more general, we search for a matching
+  # crate in all packages and use the manifest path that
+  # is referenced there.
+  cargo metadata --no-deps --format-version 1 \
+    --manifest-path "$manifest_path" \
+    | jq -r '.packages[] 
+            | select( .name == "'$expected_crate_name'") 
+            | .manifest_path'
+}
+
+# Find a Cargo.toml in the current or any sub directory
+# with a matching crate name.
+matching_cargo_toml_dir() {
+  local expected_crate_name="$1"
+
+  find -L -name Cargo.toml | sort | while read manifest_path; do
+    echo "...checking manifest_path $manifest_path" >&2
+    local matching_path="$(matching_cargo_toml_path "$manifest_path" "$expected_crate_name")"
+    if [ -n "${matching_path}" ]; then
+      echo "$(dirname $matching_path)"
+      break
+    fi
+  done
+}

pkgs/build-support/rust/build-rust-crate/log.nix

@@ -1,30 +1,56 @@
 { lib }:
-{
-  echo_build_heading = colors: ''
-    echo_build_heading() {
-     start=""
-     end=""
-     ${lib.optionalString (colors == "always") ''
-       start="$(printf '\033[0;1;32m')" #set bold, and set green.
-       end="$(printf '\033[0m')" #returns to "normal"
-     ''}
-     if (( $# == 1 )); then
-       echo "$start""Building $1""$end"
-     else
-       echo "$start""Building $1 ($2)""$end"
-     fi
+
+let echo_colored_body = start_escape:
+      # Body of a function that behaves like "echo" but 
+      # has the output colored by the given start_escape
+      # sequence. E.g.
+      #
+      # * echo_x "Building ..."
+      # * echo_x -n "Running "
+      #
+      # This is more complicated than apparent at first sight 
+      # because:
+      #   * The color markers and the text must be print
+      #     in the same echo statement. Otherise, other
+      #     intermingled text from concurrent builds will 
+      #     be colored as well.
+      #   * We need to preserve the trailing newline of the
+      #     echo if and only if it is present. Bash likes
+      #     to strip those if we capture the output of echo
+      #     in a variable.  
+      #   * Leading "-" will be interpreted by test as an
+      #     option for itself. Therefore, we prefix it with
+      #     an x in `[[ "x$1" =~ ^x- ]]`.
+      ''
+      local echo_args="";
+      while [[ "x$1" =~ ^x- ]]; do
+        echo_args+=" $1"
+        shift
+      done
+      
+      local start_escape="$(printf '${start_escape}')"
+      local reset="$(printf '\033[0m')"
+      echo $echo_args $start_escape"$@"$reset
+      '';
+  echo_conditional_colored_body = colors: start_escape:
+      if colors == "always" 
+      then (echo_colored_body start_escape)
+      else ''echo "$@"'';
+in {
+  echo_colored = colors: ''
+    echo_colored() {
+      ${echo_conditional_colored_body colors ''\033[0;1;32m''}
     }
-  '';
+
+    echo_error() {
+      ${echo_conditional_colored_body colors ''\033[0;1;31m''}
+    }
+   '';
+
   noisily = colors: verbose: ''
     noisily() {
-      start=""
-      end=""
-      ${lib.optionalString (colors == "always") ''
-        start="$(printf '\033[0;1;32m')" #set bold, and set green.
-        end="$(printf '\033[0m')" #returns to "normal"
-      ''}
   	  ${lib.optionalString verbose ''
-        echo -n "$start"Running "$end"
+        echo_colored -n "Running " 
         echo $@
   	  ''}
   	  $@

pkgs/build-support/rust/build-rust-crate/test/default.nix

@@ -8,6 +8,14 @@ let
     } // args;
   in buildRustCrate p;
 
+  mkCargoToml =
+    { name, crateVersion ? "0.1.0", path ? "Cargo.toml" }:
+      mkFile path ''
+        [package]
+        name = ${builtins.toJSON name}
+        version = ${builtins.toJSON crateVersion}
+      '';
+
   mkFile = destination: text: writeTextFile {
     name = "src";
     destination = "/${destination}";
@@ -89,7 +97,7 @@ let
   in rec {
 
   tests = let
-    cases = {
+    cases = rec {
       libPath =  { libPath = "src/my_lib.rs"; src = mkLib "src/my_lib.rs"; };
       srcLib =  { src = mkLib "src/lib.rs"; };
 
@@ -220,6 +228,40 @@ let
           ];
         };
       };
+      rustCargoTomlInSubDir = {
+        # The "workspace_member" can be set to the sub directory with the crate to build.
+        # By default ".", meaning the top level directory is assumed.
+        # Using null will trigger a search.
+        workspace_member = null;
+        src = symlinkJoin rec {
+          name = "find-cargo-toml";
+          paths = [
+            (mkCargoToml { name = "ignoreMe"; })
+            (mkTestFileWithMain "src/main.rs" "ignore_main")
+
+            (mkCargoToml { name = "rustCargoTomlInSubDir"; path = "subdir/Cargo.toml"; })
+            (mkTestFileWithMain "subdir/src/main.rs" "src_main")
+            (mkTestFile "subdir/tests/foo/main.rs" "tests_foo")
+            (mkTestFile "subdir/tests/bar/main.rs" "tests_bar")
+          ];
+        };
+        buildTests = true;
+        expectedTestOutputs = [
+          "test src_main ... ok"
+          "test tests_foo ... ok"
+          "test tests_bar ... ok"
+        ];
+      };
+
+      rustCargoTomlInTopDir =
+        let
+          withoutCargoTomlSearch = builtins.removeAttrs rustCargoTomlInSubDir [ "workspace_member" ];
+        in
+          withoutCargoTomlSearch // {
+            expectedTestOutputs = [
+              "test ignore_main ... ok"
+            ];
+          };
     };
     brotliCrates = (callPackage ./brotli-crates.nix {});
   in lib.mapAttrs (key: value: mkTest (value // lib.optionalAttrs (!value?crateName) { crateName = key; })) cases // {