lua: introduced a lua lib

Goal is to improve separation between packages and utilities.
Can help with autocompletion/navigate nixpkgs faster.
Also it will help standardize how LUA_PATH is exported across packages,
so that one can more easily make lua changes across nixpkgs (for
    instance changing where lua modules are installed).

pkgs/applications/editors/neovim/tests.nix

@@ -133,7 +133,7 @@ rec {
     configure.pathogen.pluginNames = [ "vim-nix" ];
   };
 
-  nvimWithLuaPackages = wrapNeovim2 "with-lua-packages" (makeNeovimConfig {
+  nvimWithLuaPackages = wrapNeovim2 "-with-lua-packages" (makeNeovimConfig {
     extraLuaPackages = ps: [ps.mpack];
     customRC = ''
       lua require("mpack")
@@ -141,6 +141,7 @@ rec {
   });
 
   nvim_with_lua_packages = runTest nvimWithLuaPackages ''
+    export HOME=$TMPDIR
     ${nvimWithLuaPackages}/bin/nvim -i NONE --noplugin -es
   '';
 })

pkgs/applications/editors/neovim/utils.nix

@@ -78,8 +78,7 @@ let
         ++ (extraPython3Packages ps)
         ++ (lib.concatMap (f: f ps) pluginPython3Packages));
 
-      lua = neovim-unwrapped.lua;
-      luaEnv = lua.withPackages(ps: extraLuaPackages ps);
+      luaEnv = neovim-unwrapped.lua.withPackages(extraLuaPackages);
 
       # Mapping a boolean argument to a key that tells us whether to add or not to
       # add to nvim's 'embedded rc' this:
@@ -115,8 +114,8 @@ let
         ] ++ lib.optionals (binPath != "") [
           "--suffix" "PATH" ":" binPath
         ] ++ lib.optionals (luaEnv != null) [
-          "--prefix" "LUA_PATH" ";" "${luaEnv}/share/lua/${lua.luaversion}/?.lua"
-          "--prefix" "LUA_CPATH" ";" "${luaEnv}/lib/lua/${lua.luaversion}/?.so"
+          "--prefix" "LUA_PATH" ";" (neovim-unwrapped.lua.pkgs.lib.genLuaPathAbsStr luaEnv)
+          "--prefix" "LUA_CPATH" ";" (neovim-unwrapped.lua.pkgs.lib.genLuaCPathAbsStr luaEnv)
         ];
 
 

pkgs/development/interpreters/lua-5/build-lua-package.nix

@@ -238,7 +238,7 @@ builtins.removeAttrs attrs ["disabled" "checkInputs" "externalDeps" "extraVariab
     inherit externalDeps;
   } // passthru;
 
-  meta = with lib.maintainers; {
+  meta = {
     platforms = lua.meta.platforms;
     # add extra maintainer(s) to every package
     maintainers = (meta.maintainers or []) ++ [ ];

pkgs/development/interpreters/lua-5/interpreter.nix

@@ -1,7 +1,8 @@
 { lib, stdenv, fetchurl, readline
 , compat ? false
 , callPackage
-, packageOverrides ? (self: super: {})
+, makeWrapper
+, packageOverrides ? (final: prev: {})
 , sourceVersion
 , hash
 , patches ? []
@@ -10,7 +11,9 @@
 , staticOnly ? stdenv.hostPlatform.isStatic
 }:
 let
-luaPackages = callPackage ../../lua-modules {lua=self; overrides=packageOverrides;};
+  luaPackages = callPackage ../../lua-modules {
+    lua=self; overrides=packageOverrides;
+  };
 
 plat = if stdenv.isLinux then "linux"
        else if stdenv.isDarwin then "macosx"
@@ -31,21 +34,32 @@ self = stdenv.mkDerivation rec {
     sha256 = hash;
   };
 
-  LuaPathSearchPaths    = luaPackages.getLuaPathList luaversion;
-  LuaCPathSearchPaths   = luaPackages.getLuaCPathList luaversion;
+  LuaPathSearchPaths    = luaPackages.lib.luaPathList;
+  LuaCPathSearchPaths   = luaPackages.lib.luaCPathList;
   setupHook = luaPackages.lua-setup-hook LuaPathSearchPaths LuaCPathSearchPaths;
 
+  nativeBuildInputs = [ makeWrapper ];
   buildInputs = [ readline ];
 
   inherit patches;
 
-  postPatch = lib.optionalString (!stdenv.isDarwin && !staticOnly) ''
+  # we can't pass flags to the lua makefile because for portability, everything is hardcoded
+  postPatch = ''
+    {
+      echo -e '
+        #undef  LUA_PATH_DEFAULT
+        #define LUA_PATH_DEFAULT "./share/lua/${luaversion}/?.lua;./?.lua;./?/init.lua"
+        #undef  LUA_CPATH_DEFAULT
+        #define LUA_CPATH_DEFAULT "./lib/lua/${luaversion}/?.so;./?.so;./lib/lua/${luaversion}/loadall.so"
+      '
+    } >> src/luaconf.h
+  '' + lib.optionalString (!stdenv.isDarwin && !staticOnly) ''
     # Add a target for a shared library to the Makefile.
     sed -e '1s/^/LUA_SO = liblua.so/' \
         -e 's/ALL_T *= */&$(LUA_SO) /' \
         -i src/Makefile
     cat ${./lua-dso.make} >> src/Makefile
-  '';
+  '' ;
 
   # see configurePhase for additional flags (with space)
   makeFlags = [

pkgs/development/interpreters/lua-5/wrap-lua.nix

@@ -4,16 +4,13 @@
 , makeWrapper
 }:
 
-with lib;
-
 # defined in trivial-builders.nix
 # imported as wrapLua in lua-packages.nix and passed to build-lua-derivation to be used as buildInput
 makeSetupHook {
-      deps = makeWrapper;
-      substitutions.executable = lua.interpreter;
-      substitutions.lua = lua;
-      substitutions.LuaPathSearchPaths = lib.escapeShellArgs lua.LuaPathSearchPaths;
-      substitutions.LuaCPathSearchPaths = lib.escapeShellArgs lua.LuaPathSearchPaths;
-
+  deps = makeWrapper;
+  substitutions.executable = lua.interpreter;
+  substitutions.lua = lua;
+  substitutions.LuaPathSearchPaths = lib.escapeShellArgs lua.LuaPathSearchPaths;
+  substitutions.LuaCPathSearchPaths = lib.escapeShellArgs lua.LuaPathSearchPaths;
 } ./wrap.sh
 

pkgs/development/interpreters/lua-5/wrapper.nix

@@ -11,13 +11,18 @@
 let
   env = let
     paths =  requiredLuaModules (extraLibs ++ [ lua ] );
-  in (buildEnv {
+  in buildEnv {
     name = "${lua.name}-env";
 
     inherit paths;
     inherit ignoreCollisions;
     extraOutputsToInstall = [ "out" ] ++ extraOutputsToInstall;
 
+    nativeBuildInputs = [
+      makeWrapper
+      (lua.pkgs.lua-setup-hook lua.LuaPathSearchPaths lua.LuaCPathSearchPaths)
+    ];
+
     # we create wrapper for the binaries in the different packages
     postBuild = ''
       if [ -L "$out/bin" ]; then
@@ -37,7 +42,12 @@ let
               rm -f "$out/bin/$prg"
               if [ -x "$prg" ]; then
                 nix_debug "Making wrapper $prg"
-                makeWrapper "$path/bin/$prg" "$out/bin/$prg" --suffix LUA_PATH ';' "$LUA_PATH"   --suffix LUA_CPATH ';' "$LUA_CPATH" ${lib.concatStringsSep " " makeWrapperArgs}
+                makeWrapper "$path/bin/$prg" "$out/bin/$prg" \
+                  --set-default LUA_PATH ";;" \
+                  --suffix LUA_PATH ';' "$LUA_PATH" \
+                  --set-default LUA_CPATH ";;" \
+                  --suffix LUA_CPATH ';' "$LUA_CPATH" \
+                  ${lib.concatStringsSep " " makeWrapperArgs}
               fi
             fi
           done
@@ -62,8 +72,5 @@ let
         '';
     };
     };
-  }).overrideAttrs (_: {
-    # Add extra deps needed for postBuild hook.
-    nativeBuildInputs = [ makeWrapper lua ];
-  });
+  };
 in env

pkgs/development/interpreters/luajit/default.nix

@@ -10,7 +10,7 @@
 , extraMeta ? { }
 , callPackage
 , self
-, packageOverrides ? (self: super: { })
+, packageOverrides ? (final: prev: {})
 , enableFFI ? true
 , enableJIT ? true
 , enableJITDebugModule ? enableJIT
@@ -62,6 +62,15 @@ stdenv.mkDerivation rec {
       # passed by nixpkgs CC wrapper is insufficient on its own
       substituteInPlace src/Makefile --replace "#CCDEBUG= -g" "CCDEBUG= -g"
     fi
+
+    {
+      echo -e '
+        #undef  LUA_PATH_DEFAULT
+        #define LUA_PATH_DEFAULT "./share/lua/${luaversion}/?.lua;./?.lua;./?/init.lua"
+        #undef  LUA_CPATH_DEFAULT
+        #define LUA_CPATH_DEFAULT "./lib/lua/${luaversion}/?.so;./?.so;./lib/lua/${luaversion}/loadall.so"
+      '
+    } >> src/luaconf.h
   '';
 
   configurePhase = false;
@@ -88,15 +97,10 @@ stdenv.mkDerivation rec {
     ln -s "$out"/bin/luajit-* "$out"/bin/luajit
   '';
 
-  LuaPathSearchPaths = [
-    "lib/lua/${luaversion}/?.lua"
-    "share/lua/${luaversion}/?.lua"
-    "share/lua/${luaversion}/?/init.lua"
-    "lib/lua/${luaversion}/?/init.lua"
-    "share/${name}/?.lua"
-  ];
-  LuaCPathSearchPaths = [ "lib/lua/${luaversion}/?.so" "share/lua/${luaversion}/?.so" ];
-  setupHook = luaPackages.lua-setup-hook LuaPathSearchPaths LuaCPathSearchPaths;
+  LuaPathSearchPaths    = luaPackages.lib.luaPathList;
+  LuaCPathSearchPaths   = luaPackages.lib.luaCPathList;
+
+  setupHook = luaPackages.lua-setup-hook luaPackages.lib.luaPathList luaPackages.lib.luaCPathList;
 
   passthru = rec {
     buildEnv = callPackage ../lua-5/wrapper.nix {

pkgs/development/lua-modules/default.nix

@@ -1,7 +1,7 @@
 # inspired by pkgs/development/haskell-modules/default.nix
 { pkgs, lib
 , lua
-, overrides ? (self: super: {})
+, overrides ? (final: prev: {})
 }:
 
 let
@@ -15,7 +15,7 @@ let
   overridenPackages = import ./overrides.nix { inherit pkgs; };
 
   generatedPackages = if (builtins.pathExists ./generated-packages.nix) then
-        pkgs.callPackage ./generated-packages.nix { } else (self: super: {});
+        pkgs.callPackage ./generated-packages.nix { } else (final: prev: {});
 
   extensible-self = lib.makeExtensible
     (extends overrides
@@ -24,7 +24,6 @@ let
               initialPackages
               )
           )
-    )
-          ;
+    );
 in
   extensible-self

pkgs/development/lua-modules/lib.nix

@@ -0,0 +1,63 @@
+{ pkgs, lib, lua }:
+let
+  requiredLuaModules = drvs: with lib; let
+    modules =  filter hasLuaModule drvs;
+  in unique ([lua] ++ modules ++ concatLists (catAttrs "requiredLuaModules" modules));
+  # Check whether a derivation provides a lua module.
+  hasLuaModule = drv: drv ? luaModule;
+in
+rec {
+  inherit hasLuaModule requiredLuaModules;
+
+  luaPathList = [
+    "share/lua/${lua.luaversion}/?.lua"
+    "share/lua/${lua.luaversion}/?/init.lua"
+  ];
+  luaCPathList = [
+    "lib/lua/${lua.luaversion}/?.so"
+  ];
+
+  /* generate paths without a prefix
+  */
+  luaPathRelStr = lib.concatStringsSep ";" luaPathList;
+  luaCPathRelStr = lib.concatStringsSep ";" luaCPathList;
+
+  /* generate LUA_(C)PATH value for a specific derivation, i.e., with absolute paths
+  */
+  genLuaPathAbsStr = drv: lib.concatMapStringsSep ";" (x: "${drv}/${x}") luaPathList;
+  genLuaCPathAbsStr = drv: lib.concatMapStringsSep ";" (x: "${drv}/${x}") luaCPathList;
+
+  /* Generate a LUA_PATH with absolute paths
+  */
+  # genLuaPathAbs = drv:
+  #   lib.concatStringsSep ";" (map (x: "${drv}/x") luaPathList);
+
+  luaAtLeast = lib.versionAtLeast lua.luaversion;
+  luaOlder = lib.versionOlder lua.luaversion;
+  isLua51 = (lib.versions.majorMinor lua.version) == "5.1";
+  isLua52 = (lib.versions.majorMinor lua.version) == "5.2";
+  isLua53 = lua.luaversion == "5.3";
+  isLuaJIT = lib.getName lua == "luajit";
+
+  /* generates the relative path towards the folder where
+   seems stable even when using  lua_modules_path = ""
+
+   Example:
+    getDataFolder luaPackages.stdlib
+    => stdlib-41.2.2-1-rocks/stdlib/41.2.2-1/doc
+  */
+  getDataFolder = drv:
+    "${drv.pname}-${drv.version}-rocks/${drv.pname}/${drv.version}";
+
+  /* Convert derivation to a lua module.
+    so that luaRequireModules can be run later
+  */
+  toLuaModule = drv:
+    drv.overrideAttrs( oldAttrs: {
+      # Use passthru in order to prevent rebuilds when possible.
+      passthru = (oldAttrs.passthru or {})// {
+        luaModule = lua;
+        requiredLuaModules = requiredLuaModules drv.propagatedBuildInputs;
+      };
+    });
+}

pkgs/development/lua-modules/overrides.nix

@@ -363,6 +363,12 @@ with super;
     '';
   });
 
+  # TODO just while testing, remove afterwards
+  # toVimPlugin should do it instead
+  gitsigns-nvim = super.gitsigns-nvim.overrideAttrs(oa: {
+    nativeBuildInputs = oa.nativeBuildInputs or [] ++ [ pkgs.vimUtils.vimGenDocHook ];
+  });
+
   # aliases
   cjson = super.lua-cjson;
 }

pkgs/top-level/lua-packages.nix

@@ -18,85 +18,60 @@ let
   packages = ( self:
 
 let
-  luaAtLeast = lib.versionAtLeast lua.luaversion;
-  luaOlder = lib.versionOlder lua.luaversion;
-  isLua51 = (lib.versions.majorMinor lua.version) == "5.1";
-  isLua52 = (lib.versions.majorMinor lua.version) == "5.2";
-  isLua53 = lua.luaversion == "5.3";
-  isLuaJIT = lib.getName lua == "luajit";
 
-  lua-setup-hook = callPackage ../development/interpreters/lua-5/setup-hook.nix { };
-
-  # Check whether a derivation provides a lua module.
-  hasLuaModule = drv: drv ? luaModule ;
+  # a function of lua_path / lua_cpath
+  lua-setup-hook = callPackage ../development/interpreters/lua-5/setup-hook.nix {
+    inherit lib;
+  };
 
   callPackage = pkgs.newScope self;
 
-  requiredLuaModules = drvs: with lib; let
-    modules =  filter hasLuaModule drvs;
-  in unique ([lua] ++ modules ++ concatLists (catAttrs "requiredLuaModules" modules));
-
-  # Convert derivation to a lua module.
-  toLuaModule = drv:
-    drv.overrideAttrs( oldAttrs: {
-      # Use passthru in order to prevent rebuilds when possible.
-      passthru = (oldAttrs.passthru or {})// {
-        luaModule = lua;
-        requiredLuaModules = requiredLuaModules drv.propagatedBuildInputs;
-      };
-    });
-
-
-  platformString =
-    if stdenv.isDarwin then "macosx"
-    else if stdenv.isFreeBSD then "freebsd"
-    else if stdenv.isLinux then "linux"
-    else if stdenv.isSunOS then "solaris"
-    else throw "unsupported platform";
-
   buildLuaApplication = args: buildLuarocksPackage ({namePrefix="";} // args );
 
-  buildLuarocksPackage = with pkgs.lib; makeOverridable(callPackage ../development/interpreters/lua-5/build-lua-package.nix {
-    inherit toLuaModule;
+  buildLuarocksPackage = lib.makeOverridable(callPackage ../development/interpreters/lua-5/build-lua-package.nix {
     inherit lua;
+    inherit (pkgs) lib;
+    inherit (luaLib) toLuaModule;
   });
-in
-with self; {
 
-  getLuaPathList = majorVersion: [
-    "share/lua/${majorVersion}/?.lua"
-    "share/lua/${majorVersion}/?/init.lua"
-  ];
-  getLuaCPathList = majorVersion: [
-    "lib/lua/${majorVersion}/?.so"
-  ];
-
-  # helper functions for dealing with LUA_PATH and LUA_CPATH
-  getPath = drv: pathListForVersion:
-    lib.concatMapStringsSep ";" (path: "${drv}/${path}") (pathListForVersion lua.luaversion);
-  getLuaPath = drv: getPath drv getLuaPathList;
-  getLuaCPath = drv: getPath drv getLuaCPathList;
+  luaLib = import ../development/lua-modules/lib.nix {
+    inherit (pkgs) lib;
+    inherit pkgs lua;
+  };
 
   #define build lua package function
   buildLuaPackage = callPackage ../development/lua-modules/generic {
-    inherit lua writeText;
+    inherit writeText;
   };
 
+  getPath = drv: pathListForVersion:
+    lib.concatMapStringsSep ";" (path: "${drv}/${path}") pathListForVersion;
 
-  inherit toLuaModule hasLuaModule lua-setup-hook;
-  inherit buildLuarocksPackage buildLuaApplication;
-  inherit requiredLuaModules luaOlder luaAtLeast
-    isLua51 isLua52 isLua53 isLuaJIT lua callPackage;
+in
+{
+  # helper functions for dealing with LUA_PATH and LUA_CPATH
+  lib = luaLib;
+
+  getLuaPath = drv: luaLib.getPath drv (luaLib.luaPathList lua.luaversion) ;
+  getLuaCPath = drv: luaLib.getPath drv (luaLib.luaCPathList lua.luaversion) ;
+
+
+  inherit lua lua-setup-hook callPackage;
+  inherit buildLuaPackage buildLuarocksPackage buildLuaApplication;
+  inherit (luaLib) luaOlder luaAtLeast isLua51 isLua52 isLua53 isLuaJIT
+    requiredLuaModules toLuaModule hasLuaModule;
 
   # wraps programs in $out/bin with valid LUA_PATH/LUA_CPATH
   wrapLua = callPackage ../development/interpreters/lua-5/wrap-lua.nix {
-    inherit lua; inherit (pkgs) makeSetupHook makeWrapper;
+    inherit lua lib;
+    inherit (pkgs) makeSetupHook makeWrapper;
   };
 
   luarocks = callPackage ../development/tools/misc/luarocks {
-    inherit lua;
+    inherit lua lib;
   };
 
+  # a fork of luarocks used to generate nix lua derivations from rockspecs
   luarocks-nix = callPackage ../development/tools/misc/luarocks/luarocks-nix.nix { };
 
   luxio = buildLuaPackage {
@@ -132,7 +107,7 @@ with self; {
     };
   };
 
-  vicious = toLuaModule(stdenv.mkDerivation rec {
+  vicious = toLuaModule( stdenv.mkDerivation rec {
     pname = "vicious";
     version = "2.5.0";