Merge branch 'chromium-source-refactor'

Fixes #12794 by reverting the source tree splitup (c92dbff) to use the
source tarball directly into the main Chromium derivation and making the
whole source/ subdirectory obsolete. The reasons for this are explained
in 4f981b4f84.

This also now renames the "sources.nix" file to "upstream-info.nix",
which is a more proper name for the file, because it not only contains
"source code" but also the Chrome binaries needed for the proprietary
plugins (of course "source" could also mean "where to get it", but I
wanted to avoid this ambiguity entirely).

I have successfully built and tested this using the VM tests.

All results can be found here:

https://headcounter.org/hydra/eval/313435

nixos/tests/chromium.nix

@@ -1,4 +1,11 @@
-{ system ? builtins.currentSystem }:
+{ system ? builtins.currentSystem
+, pkgs ? import ../.. { inherit system; }
+, channelMap ? {
+    stable = pkgs.chromium;
+    beta   = pkgs.chromiumBeta;
+    dev    = pkgs.chromiumDev;
+  }
+}:
 
 with import ../lib/testing.nix { inherit system; };
 with pkgs.lib;
@@ -160,8 +167,4 @@ mapAttrs (channel: chromiumPkg: makeTest rec {
 
     $machine->shutdown;
   '';
-}) {
-  stable = pkgs.chromium;
-  beta   = pkgs.chromiumBeta;
-  dev    = pkgs.chromiumDev;
-}
+}) channelMap

pkgs/applications/networking/browsers/chromium/browser.nix

@@ -29,8 +29,6 @@ mkChromiumDerivation (base: rec {
     done
   '';
 
-  preHook = "unset NIX_ENFORCE_PURITY";
-
   meta = {
     description = "An open source web browser from Google";
     homepage = http://www.chromium.org/;

pkgs/applications/networking/browsers/chromium/common.nix

@@ -1,4 +1,4 @@
-{ stdenv, fetchurl, ninja, which
+{ stdenv, ninja, which
 
 # default dependencies
 , bzip2, flac, speex, libopus
@@ -29,8 +29,7 @@
 , pulseSupport ? false, libpulseaudio ? null
 , hiDPISupport ? false
 
-, source
-, plugins
+, upstream-info
 }:
 
 buildFun:
@@ -98,9 +97,17 @@ let
 
   base = rec {
     name = "${packageName}-${version}";
-    inherit (source) version;
+    inherit (upstream-info) version;
     inherit packageName buildType buildPath;
-    src = source;
+
+    src = upstream-info.main;
+
+    unpackCmd = ''
+      tar xf "$src" \
+        --anchored \
+        --no-wildcards-match-slash \
+        --exclude='*/tools/gyp'
+    '';
 
     buildInputs = defaultDependencies ++ [
       which
@@ -118,16 +125,21 @@ let
       ++ optionals cupsSupport [ libgcrypt cups ]
       ++ optional pulseSupport libpulseaudio;
 
-    # XXX: Wait for https://crbug.com/239107 and https://crbug.com/239181 to
-    #      be fixed, then try again to unbundle everything into separate
-    #      derivations.
-    prePatch = ''
-      cp -dr --no-preserve=mode "${source.main}"/* .
-      cp -dr "${source.bundled}" third_party
-      chmod -R u+w third_party
-    '';
+    patches = [
+      ./patches/build_fixes_46.patch
+      ./patches/widevine.patch
+      (if versionOlder version "50.0.0.0"
+       then ./patches/nix_plugin_paths_46.patch
+       else ./patches/nix_plugin_paths_50.patch)
+    ];
 
     postPatch = ''
+      sed -i -r \
+        -e 's/-f(stack-protector)(-all)?/-fno-\1/' \
+        -e 's|/bin/echo|echo|' \
+        -e "/python_arch/s/: *'[^']*'/: '""'/" \
+        build/common.gypi chrome/chrome_tests.gypi
+
       sed -i -e '/module_path *=.*libexif.so/ {
         s|= [^;]*|= base::FilePath().AppendASCII("${libexif}/lib/libexif.so")|
       }' chrome/utility/media_galleries/image_metadata_extractor.cc
@@ -182,7 +194,7 @@ let
     } // (extraAttrs.gypFlags or {}));
 
     configurePhase = ''
-      # Precompile .pyc files to prevent race conditions during build
+      echo "Precompiling .py files to prevent race conditions..." >&2
       python -m compileall -q -f . > /dev/null 2>&1 || : # ignore errors
 
       # This is to ensure expansion of $out.

pkgs/applications/networking/browsers/chromium/default.nix

@@ -19,10 +19,9 @@ let
   callPackage = newScope chromium;
 
   chromium = {
-    source = callPackage ./source {
-      inherit channel;
-      # XXX: common config
-    };
+    upstream-info = (import ./update.nix {
+      inherit (stdenv) system;
+    }).getChannel channel;
 
     mkChromiumDerivation = callPackage ./common.nix {
       inherit enableSELinux enableNaCl enableHotwording gnomeSupport

pkgs/applications/networking/browsers/chromium/plugins.nix

@@ -3,7 +3,7 @@
 , enablePepperFlash ? false
 , enableWideVine ? false
 
-, source
+, upstream-info
 }:
 
 with stdenv.lib;
@@ -40,16 +40,15 @@ let
   plugins = stdenv.mkDerivation {
     name = "chromium-binary-plugins";
 
-    # XXX: Only temporary and has to be version-specific
-    src = source.plugins;
+    src = upstream-info.binary;
 
     phases = [ "unpackPhase" "patchPhase" "installPhase" "checkPhase" ];
     outputs = [ "flash" "widevine" ];
 
     unpackCmd = let
-      chan = if source.channel == "dev"    then "chrome-unstable"
-        else if source.channel == "stable" then "chrome"
-        else "chrome-${source.channel}";
+      chan = if upstream-info.channel == "dev"    then "chrome-unstable"
+        else if upstream-info.channel == "stable" then "chrome"
+        else "chrome-${upstream-info.channel}";
     in ''
       mkdir -p plugins
       ar p "$src" data.tar.xz | tar xJ -C plugins --strip-components=4 \
@@ -84,7 +83,9 @@ let
       wvModule = "@widevine@/lib/libwidevinecdmadapter.so";
       wvInfo = "#${wvName}#${wvDescription};${wvMimeTypes}";
     in ''
-      flashVersion="$(${jshon}/bin/jshon -F PepperFlash/manifest.json -e version -u)"
+      flashVersion="$(
+        "${jshon}/bin/jshon" -F PepperFlash/manifest.json -e version -u
+      )"
 
       install -vD PepperFlash/libpepflashplayer.so \
         "$flash/lib/libpepflashplayer.so"

pkgs/applications/networking/browsers/chromium/source/default.nix

@@ -1,78 +0,0 @@
-{ stdenv, fetchurl, fetchpatch, patchutils, python
-, channel ? "stable"
-}:
-
-with stdenv.lib;
-
-with (import ./update.nix {
-  inherit (stdenv) system;
-}).getChannel channel;
-
-let
-  transform = flags: concatStringsSep ";" (map (subst: subst + flags) [
-    "s,^[^/]+(.*)$,$main\\1,"
-    "s,$main/(build|tools)(/.*)?$,$out/\\1\\2,"
-    "s,$main/third_party(/.*)?$,$bundled\\1,"
-    "s,^/,,"
-  ]);
-
-in stdenv.mkDerivation {
-  name = "chromium-source-${version}";
-
-  src = fetchurl main;
-
-  buildInputs = [ python ]; # cannot patch shebangs otherwise
-
-  phases = [ "unpackPhase" "patchPhase" ];
-  outputs = [ "out" "bundled" "main" ];
-
-  unpackPhase = ''
-    tar xf "$src" -C / \
-      --transform="${transform "xS"}" \
-      --anchored \
-      --no-wildcards-match-slash \
-      --exclude='*/tools/gyp' \
-      --exclude='*/.*'
-  '';
-
-  prePatch = ''
-    for i in $outputs; do
-      eval patchShebangs "\$$i"
-    done
-  '';
-
-  patches = [
-    ./build_fixes_46.patch
-    ./widevine.patch
-    (if versionOlder version "50.0.0.0"
-     then ./nix_plugin_paths_46.patch
-     else ./nix_plugin_paths_50.patch)
-  ];
-
-  patchPhase = let
-    diffmod = sym: "/^${sym} /{s/^${sym} //;${transform ""};s/^/${sym} /}";
-    allmods = "${diffmod "---"};${diffmod "\\+\\+\\+"}";
-    sedexpr = "/^(---|\\+\\+\\+) *\\/dev\\/null/b;${allmods}";
-  in ''
-    runHook prePatch
-    for i in $patches; do
-      header "applying patch $i" 3
-      sed -r -e "${sedexpr}" "$i" | patch -d / -p0
-      stopNest
-    done
-    runHook postPatch
-  '';
-
-  postPatch = ''
-    sed -i -r \
-      -e 's/-f(stack-protector)(-all)?/-fno-\1/' \
-      -e 's|/bin/echo|echo|' \
-      -e "/python_arch/s/: *'[^']*'/: '""'/" \
-      "$out/build/common.gypi" "$main/chrome/chrome_tests.gypi"
-  '';
-
-  passthru = {
-    inherit version channel;
-    plugins = fetchurl binary;
-  };
-}

pkgs/applications/networking/browsers/chromium/update.nix

@@ -1,12 +1,12 @@
 { system ? builtins.currentSystem }:
 
 let
-  inherit (import ../../../../../../. {
+  inherit (import ../../../../../. {
     inherit system;
-  }) lib runCommand writeText stdenv curl cacert nix;
+  }) lib runCommand fetchurl writeText stdenv curl cacert nix;
 
-  sources = if builtins.pathExists ./sources.nix
-            then import ./sources.nix
+  sources = if builtins.pathExists ./upstream-info.nix
+            then import ./upstream-info.nix
             else {};
 
   bucketURL = "https://commondatastorage.googleapis.com/"
@@ -32,14 +32,15 @@ in rec {
   getChannel = channel: let
     chanAttrs = builtins.getAttr channel sources;
   in {
+    inherit channel;
     inherit (chanAttrs) version;
 
-    main = {
+    main = fetchurl {
       url = mkVerURL chanAttrs.version;
       inherit (chanAttrs) sha256;
     };
 
-    binary = let
+    binary = fetchurl (let
       mkUrls = arch: let
         mkURLForMirror = getDebURL channel chanAttrs.version arch;
       in map mkURLForMirror ([ debURL ] ++ debMirrors);
@@ -49,7 +50,7 @@ in rec {
     } else if !stdenv.is64bit && chanAttrs ? sha256bin32 then {
       urls = mkUrls "i386";
       sha256 = chanAttrs.sha256bin32;
-    } else throw "No Chrome plugins are available for your architecture.";
+    } else throw "No Chrome plugins are available for your architecture.");
   };
 
   update = let
@@ -224,8 +225,8 @@ in rec {
       mkAttr = key: val: "${mkIndent (indent + 1)}${key} = ${mkVal val};\n";
       attrLines = lib.mapAttrsToList mkAttr attrs;
     in "{\n" + (lib.concatStrings attrLines) + (mkIndent indent) + "}";
-  in writeText "chromium-new-sources.nix" ''
-    # This file is autogenerated from update.sh in the parent directory.
+  in writeText "chromium-new-upstream-info.nix" ''
+    # This file is autogenerated from update.sh in the same directory.
     ${dumpAttrs 0 newChannels}
   '';
 }

pkgs/applications/networking/browsers/chromium/update.sh

@@ -1,4 +1,4 @@
 #!/bin/sh -e
 cd "$(dirname "$0")"
-sp="$(nix-build -Q --no-out-link source/update.nix -A update)"
-cat "$sp" > source/sources.nix
+sp="$(nix-build -Q --no-out-link update.nix -A update)"
+cat "$sp" > upstream-info.nix

pkgs/applications/networking/browsers/chromium/upstream-info.nix

@@ -1,4 +1,4 @@
-# This file is autogenerated from update.sh in the parent directory.
+# This file is autogenerated from update.sh in the same directory.
 {
   beta = {
     sha256 = "1lgpjnjhy3idha5b6wp31kdk6knic96dmajyrgn1701q3mq81g1i";