dockerTools: private registry support

* authorization token is optional
* registry url is taken from X-Docker-Endpoints header
* pull.sh correctly resumes partial layer downloads
* detjson.py does not fail on missing keys

doc/functions.xml

@@ -489,7 +489,6 @@ c = lib.makeOverridable f { a = 1; b = 2; }</programlisting>
     sha256 = "1bhw5hkz6chrnrih0ymjbmn69hyfriza2lr550xyvpdrnbzr4gk2"; <co xml:id='ex-dockerTools-pullImage-4' />
 
     indexUrl = "https://index.docker.io"; <co xml:id='ex-dockerTools-pullImage-5' />
-    registryUrl = "https://registry-1.docker.io";
     registryVersion = "v1";
   }
   </programlisting>
@@ -534,8 +533,8 @@ c = lib.makeOverridable f { a = 1; b = 2; }</programlisting>
 
    <callout arearefs='ex-dockerTools-pullImage-5'>
     <para>
-     In the above example the default values are shown for the variables <varname>indexUrl</varname>,
-     <varname>registryUrl</varname> and <varname>registryVersion</varname>.
+     In the above example the default values are shown for the variables
+     <varname>indexUrl</varname> and <varname>registryVersion</varname>.
      Hence by default the Docker.io registry is used to pull the images.
     </para>
    </callout>

pkgs/build-support/docker/detjson.py

@@ -24,9 +24,11 @@ SAFEDELS["container_config"] = SAFEDELS["config"]
 
 def makedet(j, safedels):
     for k,v in safedels.items():
+        if k not in j:
+            continue
         if type(v) == dict:
             makedet(j[k], v)
-        elif k in j and j[k] == v:
+        elif j[k] == v:
             del j[k]
 
 def main():

pkgs/build-support/docker/pull.nix

@@ -8,13 +8,14 @@
 { imageName, imageTag ? "latest", imageId ? null
 , sha256, name ? "${imageName}-${imageTag}"
 , indexUrl ? "https://index.docker.io"
-, registryUrl ? "https://registry-1.docker.io"
 , registryVersion ? "v1"
 , curlOpts ? "" }:
 
+assert registryVersion == "v1";
+
 let layer = stdenv.mkDerivation {
   inherit name imageName imageTag imageId
-          indexUrl registryUrl registryVersion curlOpts;
+          indexUrl registryVersion curlOpts;
 
   builder = ./pull.sh;
   detjson = ./detjson.py;
@@ -34,10 +35,6 @@ let layer = stdenv.mkDerivation {
 
     # This variable allows the user to pass additional options to curl
     "NIX_CURL_FLAGS"
-
-    # This variable allows overriding the timeout for connecting to
-    # the hashed mirrors.
-    "NIX_CONNECT_TIMEOUT"
   ];
   
   # Doing the download on a remote machine just duplicates network

pkgs/build-support/docker/pull.sh

@@ -6,17 +6,20 @@ source $stdenv/setup
 # servers to need them during redirects, and work on SSL without a
 # certificate (this isn't a security problem because we check the
 # cryptographic hash of the output anyway).
-curl="curl \
- --location --max-redirs 20 \
- --retry 3 \
- --fail \
- --disable-epsv \
- --cookie-jar cookies \
- --insecure \
- $curlOpts \
- $NIX_CURL_FLAGS"
-
-baseUrl="$registryUrl/$registryVersion"
+curl=$(command -v curl)
+curl() {
+  [[ -n ${token:-} ]] && set -- -H "Authorization: Token $token" "$@"
+  $curl \
+    --location --max-redirs 20 \
+    --retry 3 \
+    --fail \
+    --disable-epsv \
+    --cookie-jar cookies \
+    --insecure \
+    $curlOpts \
+    $NIX_CURL_FLAGS \
+    "$@"
+}
 
 fetchLayer() {
     local url="$1"
@@ -26,7 +29,7 @@ fetchLayer() {
     # if we get error code 18, resume partial download
     while [ $curlexit -eq 18 ]; do
         # keep this inside an if statement, since on failure it doesn't abort the script
-        if $curl -H "Authorization: Token $token" "$url" --output "$dest"; then
+        if curl -C - "$url" --output "$dest"; then
             return 0
         else
             curlexit=$?;
@@ -36,17 +39,25 @@ fetchLayer() {
     return $curlexit
 }
 
-token="$($curl -o /dev/null -D- -H 'X-Docker-Token: true' "$indexUrl/$registryVersion/repositories/$imageName/images" | grep X-Docker-Token | tr -d '\r' | cut -d ' ' -f 2)"
+headers=$(curl -o /dev/null -D- -H 'X-Docker-Token: true' \
+          "$indexUrl/$registryVersion/repositories/$imageName/images")
 
-if [ -z "$token" ]; then
-    echo "error: registry returned no token"
-    exit 1
+header() {
+  grep $1 <<< "$headers" | tr -d '\r' | cut -d ' ' -f 2
+}
+
+# this only takes the first endpoint, more may be provided
+# https://docs.docker.com/v1.6/reference/api/docker-io_api/
+if ! registryUrl=$(header X-Docker-Endpoints); then
+  echo "error: index returned no endpoint"
+  exit 1
 fi
+baseUrl="https://$registryUrl/$registryVersion"
 
-# token="${token//\"/\\\"}"
+token="$(header X-Docker-Token || true)";
 
 if [ -z "$imageId" ]; then
-    imageId="$($curl -H "Authorization: Token $token" "$baseUrl/repositories/$imageName/tags/$imageTag")"
+    imageId="$(curl "$baseUrl/repositories/$imageName/tags/$imageTag")"
     imageId="${imageId//\"/}"
     if [ -z "$imageId" ]; then
         echo "error: no image ID found for ${imageName}:${imageTag}"
@@ -62,7 +73,7 @@ jshon -n object \
   -n object -s "$imageId" -i "$imageTag" \
   -i "$imageName" > $out/repositories
 
-$curl -H "Authorization: Token $token" "$baseUrl/images/$imageId/ancestry" -o ancestry.json
+curl "$baseUrl/images/$imageId/ancestry" -o ancestry.json
 
 layerIds=$(jshon -a -u < ancestry.json)
 for layerId in $layerIds; do
@@ -70,6 +81,6 @@ for layerId in $layerIds; do
     
     mkdir "$out/$layerId"
     echo '1.0' > "$out/$layerId/VERSION"
-    $curl -H "Authorization: Token $token" "$baseUrl/images/$layerId/json" | python $detjson > "$out/$layerId/json"
+    curl "$baseUrl/images/$layerId/json" | python $detjson > "$out/$layerId/json"
     fetchLayer "$baseUrl/images/$layerId/layer" "$out/$layerId/layer.tar"
 done