Merge pull request #142825 from cdepillabout/buildDhallUrl

dhallPackages.buildDhallUrl: add function for easily building dhall remote imports

doc/languages-frameworks/dhall.section.md

@@ -50,7 +50,7 @@ expression does not protect the Prelude import with a semantic integrity
 check, so the first step is to freeze the expression using `dhall freeze`,
 like this:
 
-```bash
+```ShellSession
 $ dhall freeze --inplace ./true.dhall
 ```
 
@@ -113,7 +113,7 @@ in
 
 … which we can then build using this command:
 
-```bash
+```ShellSession
 $ nix build --file ./example.nix dhallPackages.true
 ```
 
@@ -121,7 +121,7 @@ $ nix build --file ./example.nix dhallPackages.true
 
 The above package produces the following directory tree:
 
-```bash
+```ShellSession
 $ tree -a ./result
 result
 ├── .cache
@@ -135,7 +135,7 @@ result
 
 * `source.dhall` contains the result of interpreting our Dhall package:
 
-  ```bash
+  ```ShellSession
   $ cat ./result/source.dhall
   True
   ```
@@ -143,7 +143,7 @@ result
 * The `.cache` subdirectory contains one binary cache product encoding the
   same result as `source.dhall`:
 
-  ```bash
+  ```ShellSession
   $ dhall decode < ./result/.cache/dhall/122027abdeddfe8503496adeb623466caa47da5f63abd2bc6fa19f6cfcb73ecfed70
   True
   ```
@@ -151,7 +151,7 @@ result
 * `binary.dhall` contains a Dhall expression which handles fetching and decoding
   the same cache product:
 
-  ```bash
+  ```ShellSession
   $ cat ./result/binary.dhall
   missing sha256:27abdeddfe8503496adeb623466caa47da5f63abd2bc6fa19f6cfcb73ecfed70
   $ cp -r ./result/.cache .cache
@@ -168,7 +168,7 @@ to conserve disk space when they are used exclusively as dependencies.  For
 example, if we build the Prelude package it will only contain the binary
 encoding of the expression:
 
-```bash
+```ShellSession
 $ nix build --file ./example.nix dhallPackages.Prelude
 
 $ tree -a result
@@ -199,7 +199,7 @@ Dhall overlay like this:
 … and now the Prelude will contain the fully decoded result of interpreting
 the Prelude:
 
-```bash
+```ShellSession
 $ nix build --file ./example.nix dhallPackages.Prelude
 
 $ tree -a result
@@ -302,7 +302,7 @@ Additionally, `buildDhallGitHubPackage` accepts the same arguments as
 You can use the `dhall-to-nixpkgs` command-line utility to automate
 packaging Dhall code.  For example:
 
-```bash
+```ShellSession
 $ nix-env --install --attr haskellPackages.dhall-nixpkgs
 
 $ nix-env --install --attr nix-prefetch-git  # Used by dhall-to-nixpkgs
@@ -329,12 +329,12 @@ The utility takes care of automatically detecting remote imports and converting
 them to package dependencies.  You can also use the utility on local
 Dhall directories, too:
 
-```bash
+```ShellSession
 $ dhall-to-nixpkgs directory ~/proj/dhall-semver
 { buildDhallDirectoryPackage, Prelude }:
   buildDhallDirectoryPackage {
     name = "proj";
-    src = /Users/gabriel/proj/dhall-semver;
+    src = ~/proj/dhall-semver;
     file = "package.dhall";
     source = false;
     document = false;
@@ -342,6 +342,37 @@ $ dhall-to-nixpkgs directory ~/proj/dhall-semver
     }
 ```
 
+### Remote imports as fixed-output derivations {#ssec-dhall-remote-imports-as-fod}
+
+`dhall-to-nixpkgs` has the ability to fetch and build remote imports as
+fixed-output derivations by using their Dhall integrity check. This is
+sometimes easier than manually packaging all remote imports.
+
+This can be used like the following:
+
+```ShellSession
+$ dhall-to-nixpkgs directory --fixed-output-derivations ~/proj/dhall-semver
+{ buildDhallDirectoryPackage, buildDhallUrl }:
+  buildDhallDirectoryPackage {
+    name = "proj";
+    src = ~/proj/dhall-semver;
+    file = "package.dhall";
+    source = false;
+    document = false;
+    dependencies = [
+      (buildDhallUrl {
+        url = "https://prelude.dhall-lang.org/v17.0.0/package.dhall";
+        hash = "sha256-ENs8kZwl6QRoM9+Jeo/+JwHcOQ+giT2VjDQwUkvlpD4=";
+        dhallHash = "sha256:10db3c919c25e9046833df897a8ffe2701dc390fa0893d958c3430524be5a43e";
+        })
+      ];
+    }
+```
+
+Here, `dhall-semver`'s `Prelude` dependency is fetched and built with the
+`buildDhallUrl` helper function, instead of being passed in as a function
+argument.
+
 ## Overriding dependency versions {#ssec-dhall-overriding-dependency-versions}
 
 Suppose that we change our `true.dhall` example expression to depend on an older
@@ -359,7 +390,7 @@ in  Prelude.Bool.not False
 
 If we try to rebuild that expression the build will fail:
 
-```
+```ShellSession
 $ nix build --file ./example.nix dhallPackages.true
 builder for '/nix/store/0f1hla7ff1wiaqyk1r2ky4wnhnw114fi-true.drv' failed with exit code 1; last 10 log lines:
 
@@ -385,7 +416,7 @@ importing the URL.
 However, we can override the default Prelude version by using `dhall-to-nixpkgs`
 to create a Dhall package for our desired Prelude:
 
-```bash
+```ShellSession
 $ dhall-to-nixpkgs github https://github.com/dhall-lang/dhall-lang.git \
     --name Prelude \
     --directory Prelude \
@@ -396,7 +427,7 @@ $ dhall-to-nixpkgs github https://github.com/dhall-lang/dhall-lang.git \
 … and then referencing that package in our Dhall overlay, by either overriding
 the Prelude globally for all packages, like this:
 
-```bash
+```nix
   dhallOverrides = self: super: {
     true = self.callPackage ./true.nix { };
 
@@ -407,7 +438,7 @@ the Prelude globally for all packages, like this:
 … or selectively overriding the Prelude dependency for just the `true` package,
 like this:
 
-```bash
+```nix
   dhallOverrides = self: super: {
     true = self.callPackage ./true.nix {
       Prelude = self.callPackage ./Prelude.nix { };

pkgs/development/interpreters/dhall/build-dhall-url.nix

@@ -0,0 +1,96 @@
+{ cacert, dhall, dhall-docs, haskell, lib, runCommand }:
+
+# `buildDhallUrl` is similar to `buildDhallDirectoryPackage` or
+# `buildDhallGitHubPackage`, but instead builds a Nixpkgs Dhall package
+# based on a hashed URL.  This will generally be a URL that has an integrity
+# check in a Dhall file.
+#
+# Similar to `buildDhallDirectoryPackage` and `buildDhallGitHubPackage`, the output
+# of this function is a derivation that has a `binary.dhall` file, along with
+# a `.cache/` directory with the actual contents of the Dhall file from the
+# suppiled URL.
+#
+# This function is primarily used by `dhall-to-nixpkgs directory --fixed-output-derivations`.
+
+{ # URL of the input Dhall file.
+  # example: "https://raw.githubusercontent.com/cdepillabout/example-dhall-repo/c1b0d0327146648dcf8de997b2aa32758f2ed735/example1.dhall"
+  url
+
+  # Nix hash of the input Dhall file.
+  # example: "sha256-ZTSiQUXpPbPfPvS8OeK6dDQE6j6NbP27ho1cg9YfENI="
+, hash
+
+  # Dhall hash of the input Dhall file.
+  # example: "sha256:6534a24145e93db3df3ef4bc39e2ba743404ea3e8d6cfdbb868d5c83d61f10d2"
+, dhallHash
+
+  # Name for this derivation.
+, name ? (baseNameOf url + "-cache")
+
+  # `buildDhallUrl` can include both a "source distribution" in
+  # `source.dhall` and a "binary distribution" in `binary.dhall`:
+  #
+  # * `source.dhall` is a dependency-free αβ-normalized Dhall expression
+  #
+  # * `binary.dhall` is an expression of the form: `missing sha256:${HASH}`
+  #
+  #   This expression requires you to install the cache product located at
+  #   `.cache/dhall/1220${HASH}` to successfully resolve
+  #
+  # By default, `buildDhallUrl` only includes "binary.dhall" to conserve
+  # space within the Nix store, but if you set the following `source` option to
+  # `true` then the package will also include `source.dhall`.
+, source ? false
+}:
+
+let
+  # HTTP support is disabled in order to force that HTTP dependencies are built
+  # using Nix instead of using Dhall's support for HTTP imports.
+  dhallNoHTTP = haskell.lib.appendConfigureFlag dhall "-f-with-http";
+
+  # This uses Dhall's remote importing capabilities for downloading a Dhall file.
+  # The output Dhall file has all imports resolved, and then is
+  # alpha-normalized and binary-encoded.
+  downloadedEncodedFile =
+    runCommand
+      (baseNameOf url)
+      {
+        outputHashAlgo = null;
+        outputHash = hash;
+        name = baseNameOf url;
+        nativeBuildInputs = [ cacert ];
+      }
+      ''
+        echo "${url} ${dhallHash}" > in-dhall-file
+        ${dhall}/bin/dhall --alpha --plain --file in-dhall-file | ${dhallNoHTTP}/bin/dhall encode > $out
+      '';
+
+   cache = ".cache";
+
+   data = ".local/share";
+
+   cacheDhall = "${cache}/dhall";
+
+   dataDhall = "${data}/dhall";
+
+   sourceFile = "source.dhall";
+
+in
+  runCommand name { } (''
+    set -eu
+
+    mkdir -p ${cacheDhall} $out/${cacheDhall}
+
+    export XDG_CACHE_HOME=$PWD/${cache}
+
+    SHA_HASH="${dhallHash}"
+
+    HASH_FILE="''${SHA_HASH/sha256:/1220}"
+
+    cp ${downloadedEncodedFile} $out/${cacheDhall}/$HASH_FILE
+
+    echo "missing $SHA_HASH" > $out/binary.dhall
+  '' +
+  lib.optionalString source ''
+    ${dhallNoHTTP}/bin/dhall decode --file ${downloadedEncodedFile} > $out/${sourceFile}
+  '')

pkgs/test/default.nix

@@ -58,4 +58,6 @@ with pkgs;
   };
 
   writers = callPackage ../build-support/writers/test.nix {};
+
+  dhall = callPackage ./dhall { };
 }

pkgs/test/dhall/buildDhallUrl/default.nix

@@ -0,0 +1,14 @@
+{ dhallPackages, lib }:
+
+# This file tests that dhallPackages.buildDhallUrl is able to successfully
+# build a Nix Dhall package for a given remote Dhall import.
+#
+# TODO: It would be nice to extend this test to make sure that the resulting
+# Nix Dhall package is has the expected contents.
+
+dhallPackages.buildDhallUrl {
+  url = "https://raw.githubusercontent.com/cdepillabout/example-dhall-nix/e6a675c72ecd4dd23d254a02aea8181fe875747f/mydhallfile.dhall";
+  hash = "sha256-434x+QjHRzuprBdw0h6wmwB1Zj6yZqQb533me8XdO4c=";
+  dhallHash = "sha256:e37e31f908c7473ba9ac1770d21eb09b0075663eb266a41be77de67bc5dd3b87";
+  source = true;
+}

pkgs/test/dhall/default.nix

@@ -0,0 +1,5 @@
+{ lib, callPackage }:
+
+lib.recurseIntoAttrs {
+  buildDhallUrl = callPackage ./buildDhallUrl { };
+}

pkgs/top-level/dhall-packages.nix

@@ -17,12 +17,16 @@ let
       buildDhallDirectoryPackage =
         callPackage ../development/interpreters/dhall/build-dhall-directory-package.nix { };
 
+      buildDhallUrl =
+        callPackage ../development/interpreters/dhall/build-dhall-url.nix { };
+
     in
       { inherit
           callPackage
           buildDhallPackage
           buildDhallGitHubPackage
           buildDhallDirectoryPackage
+          buildDhallUrl
         ;
 
         lib = import ../development/dhall-modules/lib.nix { inherit lib; };