Merge pull request #82743 from Infinisil/partially-typed-v2

Freeform modules

lib/modules.nix

@@ -58,6 +58,23 @@ rec {
             default = check;
             description = "Whether to check whether all option definitions have matching declarations.";
           };
+
+          _module.freeformType = mkOption {
+            # Disallow merging for now, but could be implemented nicely with a `types.optionType`
+            type = types.nullOr (types.uniq types.attrs);
+            internal = true;
+            default = null;
+            description = ''
+              If set, merge all definitions that don't have an associated option
+              together using this type. The result then gets combined with the
+              values of all declared options to produce the final <literal>
+              config</literal> value.
+
+              If this is <literal>null</literal>, definitions without an option
+              will throw an error unless <option>_module.check</option> is
+              turned off.
+            '';
+          };
         };
 
         config = {
@@ -65,35 +82,44 @@ rec {
         };
       };
 
-      collected = collectModules
-        (specialArgs.modulesPath or "")
-        (modules ++ [ internalModule ])
-        ({ inherit config options lib; } // specialArgs);
+      merged =
+        let collected = collectModules
+          (specialArgs.modulesPath or "")
+          (modules ++ [ internalModule ])
+          ({ inherit lib options config; } // specialArgs);
+        in mergeModules prefix (reverseList collected);
 
-      options = mergeModules prefix (reverseList collected);
+      options = merged.matchedOptions;
 
-      # Traverse options and extract the option values into the final
-      # config set.  At the same time, check whether all option
-      # definitions have matching declarations.
-      # !!! _module.check's value can't depend on any other config values
-      # without an infinite recursion. One way around this is to make the
-      # 'config' passed around to the modules be unconditionally unchecked,
-      # and only do the check in 'result'.
-      config = yieldConfig prefix options;
-      yieldConfig = prefix: set:
-        let res = removeAttrs (mapAttrs (n: v:
-          if isOption v then v.value
-          else yieldConfig (prefix ++ [n]) v) set) ["_definedNames"];
-        in
-        if options._module.check.value && set ? _definedNames then
-          foldl' (res: m:
-            foldl' (res: name:
-              if set ? ${name} then res else throw "The option `${showOption (prefix ++ [name])}' defined in `${m.file}' does not exist.")
-              res m.names)
-            res set._definedNames
-        else
-          res;
-      result = {
+      config =
+        let
+
+          # For definitions that have an associated option
+          declaredConfig = mapAttrsRecursiveCond (v: ! isOption v) (_: v: v.value) options;
+
+          # If freeformType is set, this is for definitions that don't have an associated option
+          freeformConfig =
+            let
+              defs = map (def: {
+                file = def.file;
+                value = setAttrByPath def.prefix def.value;
+              }) merged.unmatchedDefns;
+            in if defs == [] then {}
+            else declaredConfig._module.freeformType.merge prefix defs;
+
+        in if declaredConfig._module.freeformType == null then declaredConfig
+          # Because all definitions that had an associated option ended in
+          # declaredConfig, freeformConfig can only contain the non-option
+          # paths, meaning recursiveUpdate will never override any value
+          else recursiveUpdate freeformConfig declaredConfig;
+
+      checkUnmatched =
+        if config._module.check && config._module.freeformType == null && merged.unmatchedDefns != [] then
+          let inherit (head merged.unmatchedDefns) file prefix;
+          in throw "The option `${showOption prefix}' defined in `${file}' does not exist."
+        else null;
+
+      result = builtins.seq checkUnmatched {
         inherit options;
         config = removeAttrs config [ "_module" ];
         inherit (config) _module;
@@ -174,12 +200,16 @@ rec {
   /* Massage a module into canonical form, that is, a set consisting
      of ‘options’, ‘config’ and ‘imports’ attributes. */
   unifyModuleSyntax = file: key: m:
-    let addMeta = config: if m ? meta
-      then mkMerge [ config { meta = m.meta; } ]
-      else config;
+    let
+      addMeta = config: if m ? meta
+        then mkMerge [ config { meta = m.meta; } ]
+        else config;
+      addFreeformType = config: if m ? freeformType
+        then mkMerge [ config { _module.freeformType = m.freeformType; } ]
+        else config;
     in
     if m ? config || m ? options then
-      let badAttrs = removeAttrs m ["_file" "key" "disabledModules" "imports" "options" "config" "meta"]; in
+      let badAttrs = removeAttrs m ["_file" "key" "disabledModules" "imports" "options" "config" "meta" "freeformType"]; in
       if badAttrs != {} then
         throw "Module `${key}' has an unsupported attribute `${head (attrNames badAttrs)}'. This is caused by introducing a top-level `config' or `options' attribute. Add configuration attributes immediately on the top level instead, or move all of them (namely: ${toString (attrNames badAttrs)}) into the explicit `config' attribute."
       else
@@ -188,7 +218,7 @@ rec {
           disabledModules = m.disabledModules or [];
           imports = m.imports or [];
           options = m.options or {};
-          config = addMeta (m.config or {});
+          config = addFreeformType (addMeta (m.config or {}));
         }
     else
       { _file = m._file or file;
@@ -196,7 +226,7 @@ rec {
         disabledModules = m.disabledModules or [];
         imports = m.require or [] ++ m.imports or [];
         options = {};
-        config = addMeta (removeAttrs m ["_file" "key" "disabledModules" "require" "imports"]);
+        config = addFreeformType (addMeta (removeAttrs m ["_file" "key" "disabledModules" "require" "imports" "freeformType"]));
       };
 
   applyIfFunction = key: f: args@{ config, options, lib, ... }: if isFunction f then
@@ -233,7 +263,23 @@ rec {
      declarations in all modules, combining them into a single set.
      At the same time, for each option declaration, it will merge the
      corresponding option definitions in all machines, returning them
-     in the ‘value’ attribute of each option. */
+     in the ‘value’ attribute of each option.
+
+     This returns a set like
+       {
+         # A recursive set of options along with their final values
+         matchedOptions = {
+           foo = { _type = "option"; value = "option value of foo"; ... };
+           bar.baz = { _type = "option"; value = "option value of bar.baz"; ... };
+           ...
+         };
+         # A list of definitions that weren't matched by any option
+         unmatchedDefns = [
+           { file = "file.nix"; prefix = [ "qux" ]; value = "qux"; }
+           ...
+         ];
+       }
+  */
   mergeModules = prefix: modules:
     mergeModules' prefix modules
       (concatMap (m: map (config: { file = m._file; inherit config; }) (pushDownProperties m.config)) modules);
@@ -280,9 +326,9 @@ rec {
       defnsByName' = byName "config" (module: value:
           [{ inherit (module) file; inherit value; }]
         ) configs;
-    in
-    (flip mapAttrs declsByName (name: decls:
-      # We're descending into attribute ‘name’.
+
+      resultsByName = flip mapAttrs declsByName (name: decls:
+        # We're descending into attribute ‘name’.
         let
           loc = prefix ++ [name];
           defns = defnsByName.${name} or [];
@@ -291,7 +337,10 @@ rec {
         in
           if nrOptions == length decls then
             let opt = fixupOptionType loc (mergeOptionDecls loc decls);
-            in evalOptionValue loc opt defns'
+            in {
+              matchedOptions = evalOptionValue loc opt defns';
+              unmatchedDefns = [];
+            }
           else if nrOptions != 0 then
             let
               firstOption = findFirst (m: isOption m.options) "" decls;
@@ -299,9 +348,27 @@ rec {
             in
               throw "The option `${showOption loc}' in `${firstOption._file}' is a prefix of options in `${firstNonOption._file}'."
           else
-            mergeModules' loc decls defns
-      ))
-    // { _definedNames = map (m: { inherit (m) file; names = attrNames m.config; }) configs; };
+            mergeModules' loc decls defns);
+
+      matchedOptions = mapAttrs (n: v: v.matchedOptions) resultsByName;
+
+      # an attrset 'name' => list of unmatched definitions for 'name'
+      unmatchedDefnsByName =
+        # Propagate all unmatched definitions from nested option sets
+        mapAttrs (n: v: v.unmatchedDefns) resultsByName
+        # Plus the definitions for the current prefix that don't have a matching option
+        // removeAttrs defnsByName' (attrNames matchedOptions);
+    in {
+      inherit matchedOptions;
+
+      # Transforms unmatchedDefnsByName into a list of definitions
+      unmatchedDefns = concatLists (mapAttrsToList (name: defs:
+        map (def: def // {
+          # Set this so we know when the definition first left unmatched territory
+          prefix = [name] ++ (def.prefix or []);
+        }) defs
+      ) unmatchedDefnsByName);
+    };
 
   /* Merge multiple option declarations into a single declaration.  In
      general, there should be only one declaration of each option.

lib/tests/modules.sh

@@ -210,6 +210,29 @@ checkConfigOutput "empty" config.value.foo ./declare-lazyAttrsOf.nix ./attrsOf-c
 checkConfigError 'The option value .* in .* is not of type .*' \
   config.value ./declare-int-unsigned-value.nix ./define-value-list.nix ./define-value-int-positive.nix
 
+## Freeform modules
+# Assigning without a declared option should work
+checkConfigOutput 24 config.value ./freeform-attrsOf.nix ./define-value-string.nix
+# No freeform assigments shouldn't make it error
+checkConfigOutput '{ }' config ./freeform-attrsOf.nix
+# but only if the type matches
+checkConfigError 'The option value .* in .* is not of type .*' config.value ./freeform-attrsOf.nix ./define-value-list.nix
+# and properties should be applied
+checkConfigOutput yes config.value ./freeform-attrsOf.nix ./define-value-string-properties.nix
+# Options should still be declarable, and be able to have a type that doesn't match the freeform type
+checkConfigOutput false config.enable ./freeform-attrsOf.nix ./define-value-string.nix ./declare-enable.nix
+checkConfigOutput 24 config.value ./freeform-attrsOf.nix ./define-value-string.nix ./declare-enable.nix
+# and this should work too with nested values
+checkConfigOutput false config.nest.foo ./freeform-attrsOf.nix ./freeform-nested.nix
+checkConfigOutput bar config.nest.bar ./freeform-attrsOf.nix ./freeform-nested.nix
+# Check whether a declared option can depend on an freeform-typed one
+checkConfigOutput null config.foo ./freeform-attrsOf.nix ./freeform-str-dep-unstr.nix
+checkConfigOutput 24 config.foo ./freeform-attrsOf.nix ./freeform-str-dep-unstr.nix ./define-value-string.nix
+# Check whether an freeform-typed value can depend on a declared option, this can only work with lazyAttrsOf
+checkConfigError 'infinite recursion encountered' config.foo ./freeform-attrsOf.nix ./freeform-unstr-dep-str.nix
+checkConfigError 'The option .* is used but not defined' config.foo ./freeform-lazyAttrsOf.nix ./freeform-unstr-dep-str.nix
+checkConfigOutput 24 config.foo ./freeform-lazyAttrsOf.nix ./freeform-unstr-dep-str.nix ./define-value-string.nix
+
 cat <<EOF
 ====== module tests ======
 $pass Pass

lib/tests/modules/define-value-string-properties.nix

@@ -0,0 +1,12 @@
+{ lib, ... }: {
+
+  imports = [{
+    value = lib.mkDefault "def";
+  }];
+
+  value = lib.mkMerge [
+    (lib.mkIf false "nope")
+    "yes"
+  ];
+
+}

lib/tests/modules/freeform-attrsOf.nix

@@ -0,0 +1,3 @@
+{ lib, ... }: {
+  freeformType = with lib.types; attrsOf (either str (attrsOf str));
+}

lib/tests/modules/freeform-lazyAttrsOf.nix

@@ -0,0 +1,3 @@
+{ lib, ... }: {
+  freeformType = with lib.types; lazyAttrsOf (either str (lazyAttrsOf str));
+}

lib/tests/modules/freeform-nested.nix

@@ -0,0 +1,7 @@
+{ lib, ... }: {
+  options.nest.foo = lib.mkOption {
+    type = lib.types.bool;
+    default = false;
+  };
+  config.nest.bar = "bar";
+}

lib/tests/modules/freeform-str-dep-unstr.nix

@@ -0,0 +1,8 @@
+{ lib, config, ... }: {
+  options.foo = lib.mkOption {
+    type = lib.types.nullOr lib.types.str;
+    default = null;
+  };
+
+  config.foo = lib.mkIf (config ? value) config.value;
+}

lib/tests/modules/freeform-unstr-dep-str.nix

@@ -0,0 +1,8 @@
+{ lib, config, ... }: {
+  options.value = lib.mkOption {
+    type = lib.types.nullOr lib.types.str;
+    default = null;
+  };
+
+  config.foo = lib.mkIf (config.value != null) config.value;
+}

lib/types.nix

@@ -486,9 +486,15 @@ rec {
           else value
         ) defs;
 
+        freeformType = (evalModules {
+          inherit modules specialArgs;
+          args.name = "‹name›";
+        })._module.freeformType;
+
       in
       mkOptionType rec {
         name = "submodule";
+        description = freeformType.description or name;
         check = x: isAttrs x || isFunction x || path.check x;
         merge = loc: defs:
           (evalModules {

nixos/doc/manual/development/freeform-modules.xml

@@ -0,0 +1,68 @@
+<section xmlns="http://docbook.org/ns/docbook"
+        xmlns:xlink="http://www.w3.org/1999/xlink"
+        xmlns:xi="http://www.w3.org/2001/XInclude"
+        version="5.0"
+        xml:id="sec-freeform-modules">
+ <title>Freeform modules</title>
+ <para>
+  Freeform modules allow you to define values for option paths that have not been declared explicitly. This can be used to add attribute-specific types to what would otherwise have to be <literal>attrsOf</literal> options in order to accept all attribute names.
+ </para>
+ <para>
+  This feature can be enabled by using the attribute <literal>freeformType</literal> to define a freeform type. By doing this, all assignments without an associated option will be merged using the freeform type and combined into the resulting <literal>config</literal> set. Since this feature nullifies name checking for entire option trees, it is only recommended for use in submodules.
+ </para>
+ <example xml:id="ex-freeform-module">
+  <title>Freeform submodule</title>
+  <para>
+   The following shows a submodule assigning a freeform type that allows arbitrary attributes with <literal>str</literal> values below <literal>settings</literal>, but also declares an option for the <literal>settings.port</literal> attribute to have it type-checked and assign a default value. See <xref linkend="ex-settings-typed-attrs"/> for a more complete example.
+  </para>
+ <programlisting>
+{ lib, config, ... }: {
+
+  options.settings = lib.mkOption {
+    type = lib.types.submodule {
+
+      freeformType = with lib.types; attrsOf str;
+
+      # We want this attribute to be checked for the correct type
+      options.port = lib.mkOption {
+        type = lib.types.port;
+        # Declaring the option also allows defining a default value
+        default = 8080;
+      };
+
+    };
+  };
+}
+ </programlisting>
+ <para>
+  And the following shows what such a module then allows
+ </para>
+ <programlisting>
+{
+  # Not a declared option, but the freeform type allows this
+  settings.logLevel = "debug";
+
+  # Not allowed because the the freeform type only allows strings
+  # settings.enable = true;
+
+  # Allowed because there is a port option declared
+  settings.port = 80;
+
+  # Not allowed because the port option doesn't allow strings
+  # settings.port = "443";
+}
+ </programlisting>
+ </example>
+ <note>
+  <para>
+   Freeform attributes cannot depend on other attributes of the same set without infinite recursion:
+<programlisting>
+{
+  # This throws infinite recursion encountered
+  settings.logLevel = lib.mkIf (config.settings.port == 80) "debug";
+}
+</programlisting>
+   To prevent this, declare options for all attributes that need to depend on others. For above example this means to declare <literal>logLevel</literal> to be an option.
+  </para>
+ </note>
+</section>

nixos/doc/manual/development/settings-options.xml

@@ -137,7 +137,7 @@ in {
       description = ''
         Configuration for foo, see
         <link xlink:href="https://example.com/docs/foo"/>
-        for supported values.
+        for supported settings.
       '';
     };
   };
@@ -167,13 +167,50 @@ in {
 
     # We know that the `user` attribute exists because we set a default value
     # for it above, allowing us to use it without worries here
-    users.users.${cfg.settings.user} = {}
+    users.users.${cfg.settings.user} = {};
 
     # ...
   };
 }
 </programlisting>
    </example>
+   <section xml:id="sec-settings-attrs-options">
+    <title>Option declarations for attributes</title>
+    <para>
+     Some <literal>settings</literal> attributes may deserve some extra care. They may need a different type, default or merging behavior, or they are essential options that should show their documentation in the manual. This can be done using <xref linkend='sec-freeform-modules'/>.
+     <example xml:id="ex-settings-typed-attrs">
+      <title>Declaring a type-checked <literal>settings</literal> attribute</title>
+      <para>
+       We extend above example using freeform modules to declare an option for the port, which will enforce it to be a valid integer and make it show up in the manual.
+      </para>
+<programlisting>
+settings = lib.mkOption {
+  type = lib.types.submodule {
+
+    freeformType = settingsFormat.type;
+
+    # Declare an option for the port such that the type is checked and this option
+    # is shown in the manual.
+    options.port = lib.mkOption {
+      type = lib.types.port;
+      default = 8080;
+      description = ''
+        Which port this service should listen on.
+      '';
+    };
+
+  };
+  default = {};
+  description = ''
+    Configuration for Foo, see
+    &lt;link xlink:href="https://example.com/docs/foo"/&gt;
+    for supported values.
+  '';
+};
+</programlisting>
+     </example>
+    </para>
+   </section>
  </section>
 
 </section>

nixos/doc/manual/development/writing-modules.xml

@@ -183,5 +183,6 @@ in {
  <xi:include href="meta-attributes.xml" />
  <xi:include href="importing-modules.xml" />
  <xi:include href="replace-modules.xml" />
+ <xi:include href="freeform-modules.xml" />
  <xi:include href="settings-options.xml" />
 </chapter>