Merge pull request #164662 from infinisil/fetchurl-curlOpts-list

fetchurl: Allow passing curl options with spaces

pkgs/build-support/fetchurl/builder.sh

@@ -22,6 +22,8 @@ if ! [ -f "$SSL_CERT_FILE" ]; then
     curl+=(--insecure)
 fi
 
+eval "curl+=($curlOptsList)"
+
 curl+=(
     $curlOpts
     $NIX_CURL_FLAGS

pkgs/build-support/fetchurl/default.nix

@@ -46,8 +46,13 @@ in
   urls ? []
 
 , # Additional curl options needed for the download to succeed.
+  # Warning: Each space (no matter the escaping) will start a new argument.
+  # If you wish to pass arguments with spaces, use `curlOptsList`
   curlOpts ? ""
 
+, # Additional curl options needed for the download to succeed.
+  curlOptsList ? []
+
 , # Name of the file.  If empty, use the basename of `url' (or of the
   # first element of `urls').
   name ? ""
@@ -147,7 +152,14 @@ stdenvNoCC.mkDerivation {
 
   outputHashMode = if (recursiveHash || executable) then "recursive" else "flat";
 
-  inherit curlOpts showURLs mirrorsFile postFetch downloadToTemp executable;
+  curlOpts = lib.warnIf (lib.isList curlOpts) ''
+    fetchurl for ${toString (builtins.head urls_)}: curlOpts is a list (${lib.generators.toPretty { multiline = false; } curlOpts}), which is not supported anymore.
+    - If you wish to get the same effect as before, for elements with spaces (even if escaped) to expand to multiple curl arguments, use a string argument instead:
+      curlOpts = ${lib.strings.escapeNixString (toString curlOpts)};
+    - If you wish for each list element to be passed as a separate curl argument, allowing arguments to contain spaces, use curlOptsList instead:
+      curlOptsList = [ ${lib.concatMapStringsSep " " lib.strings.escapeNixString curlOpts} ];'' curlOpts;
+  curlOptsList = lib.escapeShellArgs curlOptsList;
+  inherit showURLs mirrorsFile postFetch downloadToTemp executable;
 
   impureEnvVars = impureEnvVars ++ netrcImpureEnvVars;
 

pkgs/build-support/fetchurl/tests.nix

@@ -0,0 +1,13 @@
+{ invalidateFetcherByDrvHash, fetchurl, jq, moreutils, ... }: {
+  # Tests that we can send custom headers with spaces in them
+  header =
+    let headerValue = "Test '\" <- These are some quotes";
+    in invalidateFetcherByDrvHash fetchurl {
+      url = "https://httpbin.org/headers";
+      sha256 = builtins.hashString "sha256" (headerValue + "\n");
+      curlOptsList = [ "-H" "Hello: ${headerValue}" ];
+      postFetch = ''
+        ${jq}/bin/jq -r '.headers.Hello' $out | ${moreutils}/bin/sponge $out
+      '';
+    };
+}

pkgs/games/factorio/default.nix

@@ -85,7 +85,7 @@ let
         (lib.overrideDerivation
           (fetchurl {
             inherit name url sha256;
-            curlOpts = [
+            curlOptsList = [
               "--get"
               "--data-urlencode" "username@username"
               "--data-urlencode" "token@token"

pkgs/test/default.nix

@@ -29,6 +29,7 @@ with pkgs;
   cc-multilib-gcc = callPackage ./cc-wrapper/multilib.nix { stdenv = gccMultiStdenv; };
   cc-multilib-clang = callPackage ./cc-wrapper/multilib.nix { stdenv = clangMultiStdenv; };
 
+  fetchurl = callPackages ../build-support/fetchurl/tests.nix { };
   fetchpatch = callPackages ../build-support/fetchpatch/tests.nix { };
   fetchzip = callPackages ../build-support/fetchzip/tests.nix { };
   fetchgit = callPackages ../build-support/fetchgit/tests.nix { };