JakeHillion/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #71393 from DianaOlympos/beam-package/remove

Browse Source 
beam-modules/hex-snapshot: remove
This commit is contained in:
Jörg Thalheim 2019-11-17 13:28:13 +00:00 committed by GitHub
commit 1296f83a08
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 41 additions and 38130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

307
doc/languages-frameworks/beam.xml
View File

@ -26,7 +26,7 @@
</listitem>
<listitem>
<para>
<literal>packages</literal>: a set of package sets, each compiled with a specific Erlang/OTP version, e.g. <literal>beam.packages.erlangR19</literal>.
<literal>packages</literal>: a set of package builders (Mix and rebar3), each compiled with a specific Erlang/OTP version, e.g. <literal>beam.packages.erlangR19</literal>.
</para>
</listitem>
</itemizedlist>
@ -36,15 +36,11 @@
</para>
<para>
To create a package set built with a custom Erlang version, use the lambda, <literal>beam.packagesWith</literal>, which accepts an Erlang/OTP derivation and produces a package set similar to <literal>beam.packages.erlang</literal>.
To create a package builder built with a custom Erlang version, use the lambda, <literal>beam.packagesWith</literal>, which accepts an Erlang/OTP derivation and produces a package builder similar to <literal>beam.packages.erlang</literal>.
</para>
<para>
Many Erlang/OTP distributions available in <literal>beam.interpreters</literal> have versions with ODBC and/or Java enabled. For example, there's <literal>beam.interpreters.erlangR19_odbc_javac</literal>, which corresponds to <literal>beam.interpreters.erlangR19</literal>.
</para>
<para xml:id="erlang-call-package">
We also provide the lambda, <literal>beam.packages.erlang.callPackage</literal>, which simplifies writing BEAM package definitions by injecting all packages from <literal>beam.packages.erlang</literal> into the top-level context.
Many Erlang/OTP distributions available in <literal>beam.interpreters</literal> have versions with ODBC and/or Java enabled or without wx (no observer support). For example, there's <literal>beam.interpreters.erlangR22_odbc_javac</literal>, which corresponds to <literal>beam.interpreters.erlangR22</literal> and <literal>beam.interpreters.erlangR22_nox</literal>, which corresponds to <literal>beam.interpreters.erlangR22</literal>.
</para>
</section>
@ -55,7 +51,7 @@
<title>Rebar3</title>
<para>
We provide a version of Rebar3, which is the normal, unmodified Rebar3, under <literal>rebar3</literal>. We also provide a helper to fetch Rebar3 dependencies from a lockfile under <literal>fetchRebar3Deps</literal>.
We provide a version of Rebar3, under <literal>rebar3</literal>. We also provide a helper to fetch Rebar3 dependencies from a lockfile under <literal>fetchRebar3Deps</literal>.
</para>
</section>
@ -72,32 +68,14 @@
<title>How to Install BEAM Packages</title>
<para>
BEAM packages are not registered at the top level, simply because they are not relevant to the vast majority of Nix users. They are installable using the <literal>beam.packages.erlang</literal> attribute set (aliased as <literal>beamPackages</literal>), which points to packages built by the default Erlang/OTP version in Nixpkgs, as defined by <literal>beam.interpreters.erlang</literal>. To list the available packages in <literal>beamPackages</literal>, use the following command:
BEAM builders are not registered at the top level, simply because they are not relevant to the vast majority of Nix users.
To install any of those builders into your profile, refer to them by their attribute path <literal>beamPackages.rebar3</literal>:
</para>
<screen>
<prompt>$ </prompt>nix-env -f &quot;&lt;nixpkgs&gt;&quot; -qaP -A beamPackages
beamPackages.esqlite esqlite-0.2.1
beamPackages.goldrush goldrush-0.1.7
beamPackages.ibrowse ibrowse-4.2.2
beamPackages.jiffy jiffy-0.14.5
beamPackages.lager lager-3.0.2
beamPackages.meck meck-0.8.3
beamPackages.rebar3-pc pc-1.1.0
</screen>
<para>
To install any of those packages into your profile, refer to them by their attribute path (first column):
</para>
<screen>
<prompt>$ </prompt>nix-env -f &quot;&lt;nixpkgs&gt;&quot; -iA beamPackages.ibrowse
</screen>
<para>
The attribute path of any BEAM package corresponds to the name of that particular package in <link xlink:href="https://hex.pm">Hex</link> or its OTP Application/Release name.
</para>
</section>
<screen>
<prompt>$ </prompt>nix-env -f &quot;&lt;nixpkgs&gt;&quot; -iA beamPackages.rebar3
</screen>
</section>
<section xml:id="packaging-beam-applications">
<title>Packaging BEAM Applications</title>
@ -109,35 +87,7 @@ beamPackages.rebar3-pc pc-1.1.0
<title>Rebar3 Packages</title>
<para>
The Nix function, <literal>buildRebar3</literal>, defined in <literal>beam.packages.erlang.buildRebar3</literal> and aliased at the top level, can be used to build a derivation that understands how to build a Rebar3 project. For example, we can build <link
xlink:href="https://github.com/erlang-nix/hex2nix">hex2nix</link> as follows:
</para>
<programlisting>
{ stdenv, fetchFromGitHub, buildRebar3, ibrowse, jsx, erlware_commons }:
buildRebar3 rec {
name = "hex2nix";
version = "0.0.1";
src = fetchFromGitHub {
owner = "ericbmerritt";
repo = "hex2nix";
rev = "${version}";
sha256 = "1w7xjidz1l5yjmhlplfx7kphmnpvqm67w99hd2m7kdixwdxq0zqg";
};
beamDeps = [ ibrowse jsx erlware_commons ];
}
</programlisting>
<para>
Such derivations are callable with <literal>beam.packages.erlang.callPackage</literal> (see <xref
linkend="erlang-call-package"/>). To call this package using the normal <literal>callPackage</literal>, refer to dependency packages via <literal>beamPackages</literal>, e.g. <literal>beamPackages.ibrowse</literal>.
</para>
<para>
Notably, <literal>buildRebar3</literal> includes <literal>beamDeps</literal>, while <literal>stdenv.mkDerivation</literal> does not. BEAM dependencies added there will be correctly handled by the system.
The Nix function, <literal>buildRebar3</literal>, defined in <literal>beam.packages.erlang.buildRebar3</literal> and aliased at the top level, can be used to build a derivation that understands how to build a Rebar3 project.
</para>
<para>
@ -152,30 +102,6 @@ buildRebar3 rec {
Erlang.mk functions similarly to Rebar3, except we use <literal>buildErlangMk</literal> instead of <literal>buildRebar3</literal>.
</para>
<programlisting>
{ buildErlangMk, fetchHex, cowlib, ranch }:
buildErlangMk {
name = "cowboy";
version = "1.0.4";
src = fetchHex {
pkg = "cowboy";
version = "1.0.4";
sha256 = "6a0edee96885fae3a8dd0ac1f333538a42e807db638a9453064ccfdaa6b9fdac";
};
beamDeps = [ cowlib ranch ];
meta = {
description = ''
Small, fast, modular HTTP server written in Erlang
'';
license = stdenv.lib.licenses.isc;
homepage = https://github.com/ninenines/cowboy;
};
}
</programlisting>
</section>
<section xml:id="mix-packages">
@ -185,57 +111,9 @@ buildErlangMk {
Mix functions similarly to Rebar3, except we use <literal>buildMix</literal> instead of <literal>buildRebar3</literal>.
</para>
<programlisting>
{ buildMix, fetchHex, plug, absinthe }:
buildMix {
name = "absinthe_plug";
version = "1.0.0";
src = fetchHex {
pkg = "absinthe_plug";
version = "1.0.0";
sha256 = "08459823fe1fd4f0325a8bf0c937a4520583a5a26d73b193040ab30a1dfc0b33";
};
beamDeps = [ plug absinthe ];
meta = {
description = ''
A plug for Absinthe, an experimental GraphQL toolkit
'';
license = stdenv.lib.licenses.bsd3;
homepage = https://github.com/CargoSense/absinthe_plug;
};
}
</programlisting>
<para>
Alternatively, we can use <literal>buildHex</literal> as a shortcut:
</para>
<programlisting>
{ buildHex, buildMix, plug, absinthe }:
buildHex {
name = "absinthe_plug";
version = "1.0.0";
sha256 = "08459823fe1fd4f0325a8bf0c937a4520583a5a26d73b193040ab30a1dfc0b33";
builder = buildMix;
beamDeps = [ plug absinthe ];
meta = {
description = ''
A plug for Absinthe, an experimental GraphQL toolkit
'';
license = stdenv.lib.licenses.bsd3;
homepage = https://github.com/CargoSense/absinthe_plug;
};
}
</programlisting>
</section>
</section>
</section>
@ -243,66 +121,13 @@ buildHex {
<section xml:id="how-to-develop">
<title>How to Develop</title>
<section xml:id="accessing-an-environment">
<title>Accessing an Environment</title>
<para>
Often, we simply want to access a valid environment that contains a specific package and its dependencies. We can accomplish that with the <literal>env</literal> attribute of a derivation. For example, let's say we want to access an Erlang REPL with <literal>ibrowse</literal> loaded up. We could do the following:
</para>
<screen>
<prompt>$ </prompt><userinput>nix-shell -A beamPackages.ibrowse.env --run "erl"</userinput>
<computeroutput>Erlang/OTP 18 [erts-7.0] [source] [64-bit] [smp:4:4] [async-threads:10] [hipe] [kernel-poll:false]
Eshell V7.0 (abort with ^G)</computeroutput>
<prompt>1> </prompt><userinput>m(ibrowse).</userinput>
<computeroutput>Module: ibrowse
MD5: 3b3e0137d0cbb28070146978a3392945
Compiled: January 10 2016, 23:34
Object file: /nix/store/g1rlf65rdgjs4abbyj4grp37ry7ywivj-ibrowse-4.2.2/lib/erlang/lib/ibrowse-4.2.2/ebin/ibrowse.beam
Compiler options: [{outdir,"/tmp/nix-build-ibrowse-4.2.2.drv-0/hex-source-ibrowse-4.2.2/_build/default/lib/ibrowse/ebin"},
debug_info,debug_info,nowarn_shadow_vars,
warn_unused_import,warn_unused_vars,warnings_as_errors,
{i,"/tmp/nix-build-ibrowse-4.2.2.drv-0/hex-source-ibrowse-4.2.2/_build/default/lib/ibrowse/include"}]
Exports:
add_config/1 send_req_direct/7
all_trace_off/0 set_dest/3
code_change/3 set_max_attempts/3
get_config_value/1 set_max_pipeline_size/3
get_config_value/2 set_max_sessions/3
get_metrics/0 show_dest_status/0
get_metrics/2 show_dest_status/1
handle_call/3 show_dest_status/2
handle_cast/2 spawn_link_worker_process/1
handle_info/2 spawn_link_worker_process/2
init/1 spawn_worker_process/1
module_info/0 spawn_worker_process/2
module_info/1 start/0
rescan_config/0 start_link/0
rescan_config/1 stop/0
send_req/3 stop_worker_process/1
send_req/4 stream_close/1
send_req/5 stream_next/1
send_req/6 terminate/2
send_req_direct/4 trace_off/0
send_req_direct/5 trace_off/2
send_req_direct/6 trace_on/0
trace_on/2
ok</computeroutput>
<prompt>2></prompt>
</screen>
<para>
Notice the <literal>-A beamPackages.ibrowse.env</literal>. That is the key to this functionality.
</para>
</section>
<section xml:id="creating-a-shell">
<title>Creating a Shell</title>
<para>
Getting access to an environment often isn't enough to do real development. Usually, we need to create a <literal>shell.nix</literal> file and do our development inside of the environment specified therein. This file looks a lot like the packaging described above, except that <literal>src</literal> points to the project root and we call the package directly.
</para>
<para>
Usually, we need to create a <literal>shell.nix</literal> file and do our development inside of the environment specified therein. Just install your version of erlang and other interpreter, and then user your normal build tools.
As an example with elixir:
</para>
<programlisting>
{ pkgs ? import &quot;&lt;nixpkgs&quot;&gt; {} }:
@ -311,114 +136,24 @@ with pkgs;
let
f = { buildRebar3, ibrowse, jsx, erlware_commons }:
buildRebar3 {
name = "hex2nix";
version = "0.1.0";
src = ./.;
beamDeps = [ ibrowse jsx erlware_commons ];
};
drv = beamPackages.callPackage f {};
elixir = beam.packages.erlangR22.elixir_1_9;
in
mkShell {
buildInputs = [ elixir ];
drv
ERL_INCLUDE_PATH="${erlang}/lib/erlang/usr/include";
}
</programlisting>
<section xml:id="building-in-a-shell">
<title>Building in a Shell (for Mix Projects)</title>
<para>
We can leverage the support of the derivation, irrespective of the build derivation, by calling the commands themselves.
</para>
<programlisting>
# =============================================================================
# Variables
# =============================================================================
NIX_TEMPLATES := "$(CURDIR)/nix-templates"
TARGET := "$(PREFIX)"
PROJECT_NAME := thorndyke
NIXPKGS=../nixpkgs
NIX_PATH=nixpkgs=$(NIXPKGS)
NIX_SHELL=nix-shell -I "$(NIX_PATH)" --pure
# =============================================================================
# Rules
# =============================================================================
.PHONY= all test clean repl shell build test analyze configure install \
test-nix-install publish plt analyze
all: build
guard-%:
@ if [ "${${*}}" == "" ]; then \
echo "Environment variable $* not set"; \
exit 1; \
fi
clean:
rm -rf _build
rm -rf .cache
repl:
$(NIX_SHELL) --run "iex -pa './_build/prod/lib/*/ebin'"
shell:
$(NIX_SHELL)
configure:
$(NIX_SHELL) --command 'eval "$$configurePhase"'
build: configure
$(NIX_SHELL) --command 'eval "$$buildPhase"'
install:
$(NIX_SHELL) --command 'eval "$$installPhase"'
test:
$(NIX_SHELL) --command 'mix test --no-start --no-deps-check'
plt:
$(NIX_SHELL) --run "mix dialyzer.plt --no-deps-check"
analyze: build plt
$(NIX_SHELL) --run "mix dialyzer --no-compile"
</programlisting>
<para>
Using a <literal>shell.nix</literal> as described (see <xref
linkend="creating-a-shell"/>) should just work. Aside from <literal>test</literal>, <literal>plt</literal>, and <literal>analyze</literal>, the Make targets work just fine for all of the build derivations.
linkend="creating-a-shell"/>) should just work.
</para>
</section>
</section>
</section>
<section xml:id="generating-packages-from-hex-with-hex2nix">
<title>Generating Packages from Hex with <literal>hex2nix</literal></title>
<para>
Updating the <link xlink:href="https://hex.pm">Hex</link> package set requires <link
xlink:href="https://github.com/erlang-nix/hex2nix">hex2nix</link>. Given the path to the Erlang modules (usually <literal>pkgs/development/erlang-modules</literal>), it will dump a file called <literal>hex-packages.nix</literal>, containing all the packages that use a recognized build system in <link
xlink:href="https://hex.pm">Hex</link>. It can't be determined, however, whether every package is buildable.
</para>
<para>
To make life easier for our users, try to build every <link
xlink:href="https://hex.pm">Hex</link> package and remove those that fail. To do that, simply run the following command in the root of your <literal>nixpkgs</literal> repository:
</para>
<screen>
<prompt>$ </prompt>nix-build -A beamPackages
</screen>
<para>
That will attempt to build every package in <literal>beamPackages</literal>. Then manually remove those that fail. Hopefully, someone will improve <link
xlink:href="https://github.com/erlang-nix/hex2nix">hex2nix</link> in the future to automate the process.
</para>
</section>
</section>

5
nixos/doc/manual/release-notes/rl-2003.xml
View File

@ -141,6 +141,11 @@
Consider migrating to a different display manager such as LightDM (current default in NixOS),
SDDM, GDM, or using the startx module which uses Xinitrc.
</para>
<listitem>
<para>
The BEAM package set has been deleted. You will only find there the different interpreters.
You should now use the different build tools coming with the languages with sandbox mode disabled.
</para>
</listitem>
</itemizedlist>
</section>

4
pkgs/development/beam-modules/build-mix.nix
View File

@ -1,4 +1,4 @@
{ stdenv, writeText, elixir, erlang, hexRegistrySnapshot, hex, lib }:
{ stdenv, writeText, elixir, erlang, hex, lib }:
{ name
, version
@ -43,7 +43,7 @@ let
else setupHook;
inherit buildInputs;
propagatedBuildInputs = [ hexRegistrySnapshot hex elixir ] ++ beamDeps;
propagatedBuildInputs = [ hex elixir ] ++ beamDeps;
configurePhase = if configurePhase == null
then ''

10
pkgs/development/beam-modules/default.nix
View File

@ -14,16 +14,11 @@ let
defaultScope = mkScope self;
callPackage = drv: args: callPackageWithScope defaultScope drv args;
in
import ./hex-packages.nix {
inherit pkgs stdenv callPackage;
} // rec {
rec {
inherit callPackage erlang;
beamPackages = self;
hexRegistrySnapshot = callPackage ./hex-registry-snapshot.nix { };
rebar = callPackage ../tools/build-managers/rebar { };
rebar3-open = callPackage ../tools/build-managers/rebar3 { };
rebar3 = callPackage ../tools/build-managers/rebar3 { };
# rebar3 port compiler plugin is required by buildRebar3
@ -80,9 +75,6 @@ let
webdriver = callPackage ./webdriver {};
relxExe = callPackage ../tools/erlang/relx-exe {};
# The tool used to upgrade hex-packages.nix.
hex2nix = callPackage ../tools/erlang/hex2nix {};
# An example of Erlang/C++ package.
cuter = callPackage ../tools/erlang/cuter {};
};

37751
pkgs/development/beam-modules/hex-packages.nix generated
View File

File diff suppressed because it is too large Load Diff

27
pkgs/development/beam-modules/hex-registry-snapshot.nix
View File

@ -1,27 +0,0 @@
{stdenv, writeText, fetchFromGitHub }:
stdenv.mkDerivation rec {
pname = "hex-registry";
rev = "11d7a24e9f53f52490ce255a6248e71128e73aa1";
version = "unstable-2018-07-12";
src = fetchFromGitHub {
inherit rev;
owner = "erlang-nix";
repo = "hex-pm-registry-snapshots";
sha256 = "0dbpcrdh6jqmvnm1ysmy7ixyc95vnbqmikyx5kk77qwgyd43fqgi";
};
installPhase = ''
mkdir -p "$out/var/hex"
zcat "registry.ets.gz" > "$out/var/hex/registry.ets"
'';
setupHook = writeText "setupHook.sh" ''
export HEX_REGISTRY_SNAPSHOT="$1/var/hex/registry.ets"
'';
meta = {
platforms = stdenv.lib.platforms.unix;
};
}

6
pkgs/development/beam-modules/mix-bootstrap
View File

@ -25,8 +25,7 @@
-record(data, {version
, erl_libs
, root
, name
, registry_snapshot}).
, name}).
-define(LOCAL_HEX_REGISTRY, "registry.ets").
main(Args) ->
@ -51,8 +50,7 @@ gather_required_data_from_the_environment(_) ->
{ok, #data{ version = guard_env("version")
, erl_libs = os:getenv("ERL_LIBS", [])
, root = code:root_dir()
, name = guard_env("name")
, registry_snapshot = guard_env("HEX_REGISTRY_SNAPSHOT")}}.
, name = guard_env("name")}}.
-spec guard_env(string()) -> string().
guard_env(Name) ->

10
pkgs/development/tools/build-managers/rebar3/default.nix
View File

@ -1,12 +1,10 @@
{ stdenv, fetchFromGitHub,
fetchHex, erlang,
tree, hexRegistrySnapshot }:
tree }:
let
version = "3.12.0";
bootstrapper = ./rebar3-nix-bootstrap;
erlware_commons = fetchHex {
pkg = "erlware_commons";
version = "1.3.1";
@ -80,15 +78,9 @@ stdenv.mkDerivation rec {
sha256 = "0936ix7lfwsamssap58b265zid7x2m97azrr2qpjcln3xysd16lg";
};
inherit bootstrapper;
buildInputs = [ erlang tree ];
# TODO: Remove registry snapshot
propagatedBuildInputs = [ hexRegistrySnapshot ];
postPatch = ''
${erlang}/bin/escript ${bootstrapper} registry-only
mkdir -p _checkouts
mkdir -p _build/default/lib/

45
pkgs/development/tools/build-managers/rebar3/rebar3-nix-bootstrap
View File

@ -25,14 +25,12 @@
%%% Environment Variable
-record(data, {version
, registry_only = false
, debug_info = false
, compile_ports
, erl_libs
, plugins
, root
, name
, registry_snapshot}).
, name}).
-define(HEX_REGISTRY_PATH, ".cache/rebar3/hex/default/registry").
@ -41,30 +39,24 @@ main(Args) ->
{ok, RequiredData} = gather_required_data_from_the_environment(ArgData),
do_the_bootstrap(RequiredData).
%% @doc There are two modes 'registry_only' where the register is
%% created from hex and everything else.
-spec do_the_bootstrap(#data{}) -> ok.
do_the_bootstrap(RequiredData = #data{registry_only = true}) ->
ok = bootstrap_registry(RequiredData);
do_the_bootstrap(RequiredData) ->
ok = bootstrap_registry(RequiredData),
ok = bootstrap_configs(RequiredData),
ok = bootstrap_plugins(RequiredData),
ok = bootstrap_libs(RequiredData).
%% @doc
%% Argument parsing is super simple only because we want to keep the
%% dependencies minimal. For now there can be two entries on the
%% command line, "registry-only" and "debug-info"
%% dependencies minimal. For now there can be one entry on the
%% command line: "debug-info"
-spec parse_args([string()]) -> #data{}.
parse_args(Args0) ->
PossibleArgs = sets:from_list(["registry-only", "debug-info"]),
PossibleArgs = sets:from_list(["debug-info"]),
Args1 = sets:from_list(Args0),
Result = sets:subtract(Args1, PossibleArgs),
case sets:to_list(Result) of
[] ->
{ok, #data{registry_only = sets:is_element("registry-only", Args1),
debug_info = sets:is_element("debug-info", Args1)}};
{ok, #data{debug_info = sets:is_element("debug-info", Args1)}};
UnknownArgs ->
io:format("Unexpected command line arguments passed in: ~p~n",
[UnknownArgs]),
@ -156,30 +148,6 @@ fixup_app_name(FileName) ->
[Name, _Version, _Tag] -> Name
end.
-spec bootstrap_registry(#data{}) -> ok.
bootstrap_registry(#data{registry_snapshot = RegistrySnapshot}) ->
io:format("Bootstrapping Hex Registry for Rebar~n"),
make_sure_registry_snapshot_exists(RegistrySnapshot),
filelib:ensure_dir(?HEX_REGISTRY_PATH),
ok = case filelib:is_file(?HEX_REGISTRY_PATH) of
true ->
file:delete(?HEX_REGISTRY_PATH);
false ->
ok
end,
ok = file:make_symlink(RegistrySnapshot,
?HEX_REGISTRY_PATH).
-spec make_sure_registry_snapshot_exists(string()) -> ok.
make_sure_registry_snapshot_exists(RegistrySnapshot) ->
case filelib:is_file(RegistrySnapshot) of
true ->
ok;
false ->
stderr("Registry snapshot (~s) does not exist!", [RegistrySnapshot]),
erlang:halt(1)
end.
-spec gather_required_data_from_the_environment(#data{}) -> {ok, #data{}}.
gather_required_data_from_the_environment(ArgData) ->
{ok, ArgData#data{ version = guard_env("version")
@ -187,8 +155,7 @@ gather_required_data_from_the_environment(ArgData) ->
, plugins = get_env("buildPlugins", [])
, root = code:root_dir()
, name = guard_env("name")
, compile_ports = nix2bool(get_env("compilePorts", ""))
, registry_snapshot = guard_env("HEX_REGISTRY_SNAPSHOT")}}.
, compile_ports = nix2bool(get_env("compilePorts", ""))}}.
-spec nix2bool(any()) -> boolean().
nix2bool("1") ->

6
pkgs/top-level/all-packages.nix
View File

@ -8889,9 +8889,9 @@ in
lfe lfe_1_2;
inherit (beam.packages.erlang)
rebar rebar3-open rebar3
hexRegistrySnapshot fetchHex beamPackages
hex2nix relxExe;
rebar rebar3
fetchHex beamPackages
relxExe;
inherit (beam.packages.erlangR19) cuter;