Add an automated updater script and definitions for its use for WebKit

svn path=/nixpkgs/trunk/; revision=16816

pkgs/build-support/upstream-updater/attrset-to-dir.nix

@@ -0,0 +1,20 @@
+a :  
+a.stdenv.mkDerivation rec {
+  buildCommand = ''
+    ensureDir "$out/attributes"
+    
+  '' + (a.lib.concatStrings (map
+    (n: ''
+      ln -s "${a.writeTextFile {name=n; text=builtins.getAttr n a.theAttrSet;}}" $out/attributes/${n};
+    '')
+    (builtins.attrNames a.theAttrSet)
+  ));
+
+  name = "attribute-set";
+  meta = {
+    description = "Contents of an attribute set";
+    maintainers = [
+      a.lib.maintainers.raskin
+    ];
+  };
+}

pkgs/build-support/upstream-updater/attrset-to-dir.sh

@@ -0,0 +1,21 @@
+#! /bin/sh
+
+[ -n "$2" ] && NIXPKGS_ALL="$2";
+[ -z "$NIXPKGS_ALL" ] && [ -f "/etc/nixos/nixpkgs" ] && NIXPKGS_ALL="/etc/nixos/nixpkgs";
+[ -z "$NIXPKGS_ALL" ] && [ -f "$HOME/nixpkgs" ] && NIXPKGS_ALL="$HOME/nixpkgs";
+[ -z "$NIXPKGS_ALL" ] && {
+  echo "Cannot find Nixpkgs source. Please specify it via NIXPKGS_ALL or second command line argument"
+  exit 1
+};
+
+derivation="$(nix-instantiate --show-trace - << EOF
+let 
+  pkgs = import "${NIXPKGS_ALL}" {};
+  attrSet = import "${1}";
+in
+  pkgs.attrSetToDir attrSet
+EOF
+)"
+echo "Derivation is: $derivation" >&2
+output="$(nix-store -r "$derivation")"
+echo "$output/attributes"

pkgs/build-support/upstream-updater/design.txt

@@ -0,0 +1,26 @@
+Next to file.nix we get src-for-file.nix 
+src-for-file.nix should evaluate to a flat attribute set with 
+string values.
+It is supposed to be imported in the main expression.
+In the ideal world it can export url, hash, version.
+
+src-for-file.nix generation is directed by 
+src-info-for-file.nix.
+
+Attributes:
+
+src-info-for-file.nix:
+
+downloadPage
+sourceRegexp (default = '.*[.]tar[.].*')
+choiceCommand (default = 'head -1')
+versionExtractorSedScript (default = 's/.*-([0-9.a-z]+)[.].*/\1/')
+versionReferenceCreator (default = 's/-([0-9.a-z]+)[.]/-${version}./')
+mirrorSedScript (default = none)
+
+src-for-file.nix:
+
+advertisedUrl (its match is the check for update presence)
+url
+hash
+version

pkgs/build-support/upstream-updater/test-case.nix

@@ -0,0 +1,13 @@
+{
+  a=1;
+  b="text";
+  c=''
+  text
+  '';
+  d=''
+    Multi-line text with special characters - 
+    like \ (backslash) and ''${} (dollar + 
+    curly braces) and $ (dollar) and ' (quote)
+    and " (double quote).
+  '';
+}

pkgs/build-support/upstream-updater/update-upstream-data.sh

@@ -0,0 +1,51 @@
+#! /bin/sh
+
+own_dir="$(cd "$(dirname "$0")"; sh -c pwd)"
+
+main_file="$1"
+main_dir="$(cd "$(dirname "$main_file")" ; sh -c pwd)"
+file_name="$(basename "$main_file")"
+defs_file="$main_dir"/src-info-for-"$file_name"
+src_file="$main_dir"/src-for-"$file_name"
+new_src_file="$main_dir"/updated-src-for-"$file_name"
+
+defs_dir="$("$own_dir"/attrset-to-dir.sh "$defs_file")"
+src_defs_dir="$("$own_dir"/attrset-to-dir.sh "$src_file")"
+
+getAttr () {
+    file="$defs_dir"/"$1"
+    data="$( ( [ -f "$file" ] && cat "$file" ) || echo "$2" )"
+    echo "attribute $1 obtained as: [[$data]]" >&2
+    echo "$data"
+}
+
+freshUrl="$("$own_dir"/urls-from-page.sh "$(getAttr downloadPage)" |
+  egrep "$(getAttr sourceRegexp '.*[.]tar[.].*')" | 
+  sh -c "$(getAttr choiceCommand 'head -1')")"
+
+echo "Found download link: $freshUrl" >&2
+
+if [ x"$freshUrl" = x"$(cat "$src_defs_dir"/advertisedUrl)" ]; then
+    echo "Source link not changed" >&2
+    exit
+fi
+
+version="$(echo "$freshUrl" | 
+  sed -re "$(getAttr versionExtractorSedScript \
+    's/.*-([0-9.]+)[.].*/\1/')")"
+
+mirrorUrl="$(echo "$freshUrl" | 
+  sed -r -e "$(getAttr versionReferenceCreator \
+    's/-([0-9.]+)[.]/-${version}./')" |
+  sed -r -e "$(getAttr mirrorSedScript)")"
+
+hash=$(nix-prefetch-url "$freshUrl")
+
+cat << EOF > "$new_src_file"
+rec {
+  advertisedUrl="$freshUrl";
+  version = "$version";
+  url="$mirrorUrl";
+  hash = "$hash";
+}
+EOF

pkgs/build-support/upstream-updater/urls-from-page.sh

@@ -0,0 +1,12 @@
+#! /bin/sh
+
+url="$1"
+protocol="${url%%:*}"
+path="${url#$protocol://}"
+server="${path%%/*}"
+relpath="${path#$server}"
+ 
+echo "URL: $url" >&2
+
+curl -k "$url" | sed -re 's/^/-/;s/[hH][rR][eE][fF]="([^"]*)"/\n+\1\n-/g' | \
+  sed -e '/^-/d; s/^[+]//; /^#/d;'"s/^\\//$protocol:\\/\\/$server\\//g"

pkgs/development/libraries/webkit/default.nix

@@ -1,9 +1,12 @@
 args : with args; 
-let version = lib.attrByPath ["version"] "r46809" args; in
+let 
+  s = import ./src-for-default.nix;
+  version = lib.attrByPath ["version"] s.version args;
+in
 rec {
   src = fetchurl {
-    url = "http://nightly.webkit.org/files/trunk/src/WebKit-${version}.tar.bz2";
-    sha256 = "12isv3rjvjfn45mgp42nsv812cmfcfrpgbgzqgf88qyldcmq0qs5";
+    url = s.url;
+    sha256 = s.hash;
   };
 
   buildInputs = [gtk atk cairo curl fontconfig freetype

pkgs/development/libraries/webkit/src-for-default.nix

@@ -0,0 +1,6 @@
+rec {
+  advertisedUrl="http://builds.nightly.webkit.org/files/trunk/src/WebKit-r47686.tar.bz2";
+  version = "r47686";
+  url="http://builds.nightly.webkit.org/files/trunk/src/WebKit-${version}.tar.bz2";
+  hash = "1h1frvvp8hfv3q2drjnrpgadgin55p7nv8747kxxhdkj4gv9dzqg";
+}

pkgs/development/libraries/webkit/src-info-for-default.nix

@@ -0,0 +1,5 @@
+{
+  downloadPage = "http://nightly.webkit.org/";
+  versionExtractorSedScript = "s/.*-(r[0-9]+)[.].*/\\1/";
+  versionReferenceCreator = "s/-(r[0-9.]+)[.]/-\${version}./";
+}

pkgs/top-level/all-packages.nix

@@ -227,6 +227,10 @@ let
 
   ### BUILD SUPPORT
 
+  attrSetToDir = arg : import ../build-support/upstream-updater/attrset-to-dir.nix {
+    inherit writeTextFile stdenv lib;
+    theAttrSet = arg;
+  };
 
   buildEnv = import ../build-support/buildenv {
     inherit stdenv perl;