diff --git a/pkgs/development/tools/yarn2nix/bin/yarn2nix.js b/pkgs/development/tools/yarn2nix/bin/yarn2nix.js new file mode 100755 index 000000000000..3eadea559032 --- /dev/null +++ b/pkgs/development/tools/yarn2nix/bin/yarn2nix.js @@ -0,0 +1,144 @@ +#!/usr/bin/env node +"use strict"; + +const crypto = require('crypto'); +const fs = require("fs"); +const https = require("https"); +const path = require("path"); +const util = require("util"); + +const lockfile = require("@yarnpkg/lockfile") +const docopt = require("docopt").docopt; + +//////////////////////////////////////////////////////////////////////////////// + +const USAGE = ` +Usage: yarn2nix [options] + +Options: + -h --help Shows this help. + --no-nix Hide the nix output + --no-patch Don't patch the lockfile if hashes are missing + --lockfile=FILE Specify path to the lockfile [default: ./yarn.lock]. +` + +const HEAD = ` +{fetchurl, linkFarm}: rec { + offline_cache = linkFarm "offline" packages; + packages = [ +`.trim(); + +//////////////////////////////////////////////////////////////////////////////// + +function generateNix(lockedDependencies) { + let found = {}; + + console.log(HEAD) + + for (var depRange in lockedDependencies) { + let dep = lockedDependencies[depRange]; + + let depRangeParts = depRange.split('@'); + let [url, sha1] = dep["resolved"].split("#"); + let file_name = path.basename(url) + + if (found.hasOwnProperty(file_name)) { + continue; + } else { + found[file_name] = null; + } + + + console.log(` + { + name = "${file_name}"; + path = fetchurl { + name = "${file_name}"; + url = "${url}"; + sha1 = "${sha1}"; + }; + }`) + } + + console.log(" ];") + console.log("}") +} + + +function getSha1(url) { + return new Promise((resolve, reject) => { + https.get(url, (res) => { + const { statusCode } = res; + const hash = crypto.createHash('sha1'); + if (statusCode !== 200) { + const err = new Error('Request Failed.\n' + + `Status Code: ${statusCode}`); + // consume response data to free up memory + res.resume(); + reject(err); + } + + res.on('data', (chunk) => { hash.update(chunk); }); + res.on('end', () => { resolve(hash.digest('hex')) }); + res.on('error', reject); + }); + }); +}; + +function updateResolvedSha1(pkg) { + // local dependency + if (!pkg.resolved) { return Promise.resolve(); } + let [url, sha1] = pkg.resolved.split("#", 2) + if (!sha1) { + return new Promise((resolve, reject) => { + getSha1(url).then(sha1 => { + pkg.resolved = `${url}#${sha1}`; + resolve(); + }).catch(reject); + }); + } else { + // nothing to do + return Promise.resolve(); + }; +} + +function values(obj) { + var entries = []; + for (let key in obj) { + entries.push(obj[key]); + } + return entries; +} + +//////////////////////////////////////////////////////////////////////////////// +// Main +//////////////////////////////////////////////////////////////////////////////// + +var options = docopt(USAGE); + +let data = fs.readFileSync(options['--lockfile'], 'utf8') +let json = lockfile.parse(data) +if (json.type != "success") { + throw new Error("yarn.lock parse error") +} + +// Check fore missing hashes in the yarn.lock and patch if necessary +var pkgs = values(json.object); +Promise.all(pkgs.map(updateResolvedSha1)).then(() => { + let newData = lockfile.stringify(json.object); + + if (newData != data) { + console.error("found changes in the lockfile", options["--lockfile"]); + + if (options["--no-patch"]) { + console.error("...aborting"); + process.exit(1); + } + + fs.writeFileSync(options['--lockfile'], newData); + } + + if (!options['--no-nix']) { + generateNix(json.object); + } +}) diff --git a/pkgs/development/tools/yarn2nix/default.nix b/pkgs/development/tools/yarn2nix/default.nix new file mode 100644 index 000000000000..e86c0aac2c99 --- /dev/null +++ b/pkgs/development/tools/yarn2nix/default.nix @@ -0,0 +1,199 @@ +{ stdenv, lib, fetchurl, linkFarm, runCommand, nodejs, yarn }: + +let + unlessNull = item: alt: + if item == null then alt else item; + + yarn2nix = mkYarnPackage { + src = ./.; + yarnNix = ./yarn.nix; + + passthru = { + inherit + defaultYarnFlags + linkNodeModulesHook + mkYarnModules + mkYarnNix + mkYarnPackage + # Export yarn again to make it easier to find out which yarn was used. + yarn + ; + }; + + meta = with lib; { + description = "generate nix expressions from a yarn.lock file"; + homepage = "https://github.com/moretea/yarn2nix"; + license = licenses.gpl3; + maintainers = with maintainers; [ manveru zimbatm ]; + }; + }; + + # Generates the yarn.nix from the yarn.lock file + mkYarnNix = yarnLock: + runCommand "yarn.nix" {} + "${yarn2nix}/bin/yarn2nix --lockfile ${yarnLock} --no-patch > $out"; + + # Loads the generated offline cache. This will be used by yarn as + # the package source. + importOfflineCache = yarnNix: + let + pkg = import yarnNix { inherit fetchurl linkFarm; }; + in + pkg.offline_cache; + + defaultYarnFlags = [ + "--offline" + "--frozen-lockfile" + "--ignore-engines" + "--ignore-scripts" + ]; + + mkYarnModules = { + name, + packageJSON, + yarnLock, + yarnNix ? mkYarnNix yarnLock, + yarnFlags ? defaultYarnFlags, + pkgConfig ? {}, + preBuild ? "", + }: + let + offlineCache = importOfflineCache yarnNix; + extraBuildInputs = (lib.flatten (builtins.map (key: + pkgConfig.${key} . buildInputs or [] + ) (builtins.attrNames pkgConfig))); + postInstall = (builtins.map (key: + if (pkgConfig.${key} ? postInstall) then + '' + for f in $(find -L -path '*/node_modules/${key}' -type d); do + (cd "$f" && (${pkgConfig.${key}.postInstall})) + done + '' + else + "" + ) (builtins.attrNames pkgConfig)); + in + stdenv.mkDerivation { + inherit name preBuild; + phases = ["configurePhase" "buildPhase"]; + buildInputs = [ yarn nodejs ] ++ extraBuildInputs; + + configurePhase = '' + # Yarn writes cache directories etc to $HOME. + export HOME=$PWD/yarn_home + ''; + + buildPhase = '' + runHook preBuild + + cp ${packageJSON} ./package.json + cp ${yarnLock} ./yarn.lock + chmod +w ./yarn.lock + + yarn config --offline set yarn-offline-mirror ${offlineCache} + + # Do not look up in the registry, but in the offline cache. + # TODO: Ask upstream to fix this mess. + sed -i -E 's|^(\s*resolved\s*")https?://.*/|\1|' yarn.lock + yarn install ${lib.escapeShellArgs yarnFlags} + + ${lib.concatStringsSep "\n" postInstall} + + mkdir $out + mv node_modules $out/ + patchShebangs $out + ''; + }; + + # This can be used as a shellHook in mkYarnPackage. It brings the built node_modules into + # the shell-hook environment. + linkNodeModulesHook = '' + if [[ -d node_modules || -L node_modules ]]; then + echo "./node_modules is present. Replacing." + rm -rf node_modules + fi + + ln -s "$node_modules" node_modules + ''; + + mkYarnPackage = { + name ? null, + src, + packageJSON ? src + "/package.json", + yarnLock ? src + "/yarn.lock", + yarnNix ? mkYarnNix yarnLock, + yarnFlags ? defaultYarnFlags, + yarnPreBuild ? "", + pkgConfig ? {}, + extraBuildInputs ? [], + publishBinsFor ? null, + ... + }@attrs: + let + package = lib.importJSON packageJSON; + pname = package.name; + version = package.version; + deps = mkYarnModules { + name = "${pname}-modules-${version}"; + preBuild = yarnPreBuild; + inherit packageJSON yarnLock yarnNix yarnFlags pkgConfig; + }; + publishBinsFor_ = unlessNull publishBinsFor [pname]; + in stdenv.mkDerivation (builtins.removeAttrs attrs ["pkgConfig"] // { + inherit src; + + name = unlessNull name "${pname}-${version}"; + + buildInputs = [ yarn nodejs ] ++ extraBuildInputs; + + node_modules = deps + "/node_modules"; + + configurePhase = attrs.configurePhase or '' + runHook preConfigure + + if [ -d npm-packages-offline-cache ]; then + echo "npm-pacakges-offline-cache dir present. Removing." + rm -rf npm-packages-offline-cache + fi + + if [[ -d node_modules || -L node_modules ]]; then + echo "./node_modules is present. Removing." + rm -rf node_modules + fi + + mkdir -p node_modules + ln -s $node_modules/* node_modules/ + ln -s $node_modules/.bin node_modules/ + + if [ -d node_modules/${pname} ]; then + echo "Error! There is already an ${pname} package in the top level node_modules dir!" + exit 1 + fi + + runHook postConfigure + ''; + + # Replace this phase on frontend packages where only the generated + # files are an interesting output. + installPhase = attrs.installPhase or '' + runHook preInstall + + mkdir -p $out + cp -r node_modules $out/node_modules + cp -r . $out/node_modules/${pname} + rm -rf $out/node_modules/${pname}/node_modules + + mkdir $out/bin + node ${./fixup_bin.js} $out ${lib.concatStringsSep " " publishBinsFor_} + + runHook postInstall + ''; + + passthru = { + inherit package deps; + } // (attrs.passthru or {}); + + # TODO: populate meta automatically + }); +in + yarn2nix diff --git a/pkgs/development/tools/yarn2nix/fixup_bin.js b/pkgs/development/tools/yarn2nix/fixup_bin.js new file mode 100644 index 000000000000..dab1759c2046 --- /dev/null +++ b/pkgs/development/tools/yarn2nix/fixup_bin.js @@ -0,0 +1,45 @@ +#!/usr/bin/env node +"use strict"; + +/* Usage: + * node fixup_bin.js [, ... ] + */ + +const fs = require("fs"); +const path = require("path"); + +const output = process.argv[2]; +const packages_to_publish_bin = process.argv.slice(3); +const derivation_bin_path = output + "/bin"; + +function processPackage(name) { + console.log("Processing ", name); + const package_path = output + "/node_modules/" + name; + const package_json_path = package_path + "/package.json"; + const package_json = JSON.parse(fs.readFileSync(package_json_path)); + + if (!package_json.bin) { + console.log("No binaries provided"); + return; + } + + // There are two alternative syntaxes for `bin` + // a) just a plain string, in which case the name of the package is the name of the binary. + // b) an object, where key is the name of the eventual binary, and the value the path to that binary. + if (typeof package_json.bin == "string") { + let bin_name = package_json.bin; + package_json.bin = { }; + package_json.bin[package_json.name] = bin_name; + } + + for (let binName in package_json.bin) { + const bin_path = package_json.bin[binName]; + const full_bin_path = path.normalize(package_path + "/" + bin_path); + fs.symlinkSync(full_bin_path, derivation_bin_path + "/"+ binName); + console.log("Linked", binName); + } +} + +packages_to_publish_bin.forEach((pkg) => { + processPackage(pkg); +}); diff --git a/pkgs/development/tools/yarn2nix/package.json b/pkgs/development/tools/yarn2nix/package.json new file mode 100644 index 000000000000..130eee67c560 --- /dev/null +++ b/pkgs/development/tools/yarn2nix/package.json @@ -0,0 +1,19 @@ +{ + "name": "yarn2nix", + "version": "1.0.0", + "description": "Convert packages.json and yarn.lock into a Nix expression that downloads all the dependencies", + "main": "index.js", + "repository": ".", + "author": "Maarten Hoogendoorn ", + "license": "MIT", + "scripts": { + "yarn2nix": "bin/yarn2nix.js" + }, + "bin": { + "yarn2nix": "bin/yarn2nix.js" + }, + "dependencies": { + "@yarnpkg/lockfile": "^1.0.0", + "docopt": "^0.6.2" + } +} diff --git a/pkgs/development/tools/yarn2nix/yarn.lock b/pkgs/development/tools/yarn2nix/yarn.lock new file mode 100644 index 000000000000..976d2c530e1e --- /dev/null +++ b/pkgs/development/tools/yarn2nix/yarn.lock @@ -0,0 +1,11 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +"@yarnpkg/lockfile@^1.0.0": + version "1.0.0" + resolved "https://registry.yarnpkg.com/@yarnpkg/lockfile/-/lockfile-1.0.0.tgz#33d1dbb659a23b81f87f048762b35a446172add3" + +docopt@^0.6.2: + version "0.6.2" + resolved "https://registry.yarnpkg.com/docopt/-/docopt-0.6.2.tgz#b28e9e2220da5ec49f7ea5bb24a47787405eeb11" \ No newline at end of file diff --git a/pkgs/development/tools/yarn2nix/yarn.nix b/pkgs/development/tools/yarn2nix/yarn.nix new file mode 100644 index 000000000000..a9e42e7d986c --- /dev/null +++ b/pkgs/development/tools/yarn2nix/yarn.nix @@ -0,0 +1,23 @@ +{fetchurl, linkFarm}: rec { + offline_cache = linkFarm "offline" packages; + packages = [ + + { + name = "lockfile-1.0.0.tgz"; + path = fetchurl { + name = "lockfile-1.0.0.tgz"; + url = "https://registry.yarnpkg.com/@yarnpkg/lockfile/-/lockfile-1.0.0.tgz"; + sha1 = "33d1dbb659a23b81f87f048762b35a446172add3"; + }; + } + + { + name = "docopt-0.6.2.tgz"; + path = fetchurl { + name = "docopt-0.6.2.tgz"; + url = "https://registry.yarnpkg.com/docopt/-/docopt-0.6.2.tgz"; + sha1 = "b28e9e2220da5ec49f7ea5bb24a47787405eeb11"; + }; + } + ]; +} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 39cc942bc7a4..35ebbf4be77b 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -5603,6 +5603,9 @@ with pkgs; yarn = callPackage ../development/tools/yarn { }; + yarn2nix = callPackage ../development/tools/yarn2nix { }; + inherit (yarn2nix) mkYarnPackage; + yasr = callPackage ../applications/audio/yasr { }; yank = callPackage ../tools/misc/yank { };