From 0852dc859ea344d9e127c9160f0aec523218cfe5 Mon Sep 17 00:00:00 2001 From: KFears Date: Mon, 19 Sep 2022 19:16:55 +0400 Subject: [PATCH] nixos/grafana: refactor datasources for RFC42 This commit refactors `services.grafana.provision.datasources` towards the RFC42 style. To preserve backwards compatibility, we have to jump through a ton of hoops, introducing esoteric type signatures and bizarre structs. The Grafana module definition should hopefully become a lot cleaner after a release cycle or two once the old configuration style is completely deprecated. --- .../from_md/release-notes/rl-2211.section.xml | 10 +- .../manual/release-notes/rl-2211.section.md | 2 +- nixos/modules/services/monitoring/grafana.nix | 200 ++++++++++++------ nixos/tests/grafana/default.nix | 1 + .../grafana/provision-datasources/default.nix | 95 +++++++++ .../provision-datasources.yaml | 7 + 6 files changed, 242 insertions(+), 73 deletions(-) create mode 100644 nixos/tests/grafana/provision-datasources/default.nix create mode 100644 nixos/tests/grafana/provision-datasources/provision-datasources.yaml diff --git a/nixos/doc/manual/from_md/release-notes/rl-2211.section.xml b/nixos/doc/manual/from_md/release-notes/rl-2211.section.xml index de6142a0957f..808f1b5a6829 100644 --- a/nixos/doc/manual/from_md/release-notes/rl-2211.section.xml +++ b/nixos/doc/manual/from_md/release-notes/rl-2211.section.xml @@ -835,11 +835,13 @@ - The services.grafana.provision.dashboards - option was converted to a + The services.grafana.provision.datasources + and services.grafana.provision.dashboards + options were converted to a RFC - 0042 configuration. It also now supports specifying the - provisioning YAML file with path option. + 0042 configuration. They also now support specifying + the provisioning YAML file with path + option. diff --git a/nixos/doc/manual/release-notes/rl-2211.section.md b/nixos/doc/manual/release-notes/rl-2211.section.md index b06d73b2fdeb..447fe19878e9 100644 --- a/nixos/doc/manual/release-notes/rl-2211.section.md +++ b/nixos/doc/manual/release-notes/rl-2211.section.md @@ -272,7 +272,7 @@ Available as [services.patroni](options.html#opt-services.patroni.enable). - The `services.matrix-synapse` systemd unit has been hardened. -- The `services.grafana.provision.dashboards` option was converted to a [RFC 0042](https://github.com/NixOS/rfcs/blob/master/rfcs/0042-config-option.md) configuration. It also now supports specifying the provisioning YAML file with `path` option. +- The `services.grafana.provision.datasources` and `services.grafana.provision.dashboards` options were converted to a [RFC 0042](https://github.com/NixOS/rfcs/blob/master/rfcs/0042-config-option.md) configuration. They also now support specifying the provisioning YAML file with `path` option. - Matrix Synapse now requires entries in the `state_group_edges` table to be unique, in order to prevent accidentally introducing duplicate information (for example, because a database backup was restored multiple times). If your Synapse database already has duplicate rows in this table, this could fail with an error and require manual remediation. diff --git a/nixos/modules/services/monitoring/grafana.nix b/nixos/modules/services/monitoring/grafana.nix index 835389b5ca41..2cea038b8614 100644 --- a/nixos/modules/services/monitoring/grafana.nix +++ b/nixos/modules/services/monitoring/grafana.nix @@ -78,7 +78,8 @@ let datasources = cfg.provision.datasources; }; - datasourceFile = pkgs.writeText "datasource.yaml" (builtins.toJSON datasourceConfiguration); + datasourceFileNew = if (cfg.provision.datasources.path == null) then provisioningSettingsFormat.generate "datasource.yaml" cfg.provision.datasources.settings else cfg.provision.datasources.path; + datasourceFile = if (builtins.isList cfg.provision.datasources) then provisioningSettingsFormat.generate "datasource.yaml" datasourceConfiguration else datasourceFileNew; dashboardConfiguration = { apiVersion = 1; @@ -107,6 +108,8 @@ let # http://docs.grafana.org/administration/provisioning/#datasources grafanaTypes.datasourceConfig = types.submodule { + freeformType = provisioningSettingsFormat.type; + options = { name = mkOption { type = types.str; @@ -121,11 +124,6 @@ let default = "proxy"; description = lib.mdDoc "Access mode. proxy or direct (Server or Browser in the UI). Required."; }; - orgId = mkOption { - type = types.int; - default = 1; - description = lib.mdDoc "Org id. will default to orgId 1 if not specified."; - }; uid = mkOption { type = types.nullOr types.str; default = null; @@ -133,68 +131,47 @@ let }; url = mkOption { type = types.str; + default = "localhost"; description = lib.mdDoc "Url of the datasource."; }; - password = mkOption { - type = types.nullOr types.str; - default = null; - description = lib.mdDoc "Database password, if used."; - }; - user = mkOption { - type = types.nullOr types.str; - default = null; - description = lib.mdDoc "Database user, if used."; - }; - database = mkOption { - type = types.nullOr types.str; - default = null; - description = lib.mdDoc "Database name, if used."; - }; - basicAuth = mkOption { - type = types.nullOr types.bool; - default = null; - description = lib.mdDoc "Enable/disable basic auth."; - }; - basicAuthUser = mkOption { - type = types.nullOr types.str; - default = null; - description = lib.mdDoc "Basic auth username."; - }; - basicAuthPassword = mkOption { - type = types.nullOr types.str; - default = null; - description = lib.mdDoc "Basic auth password."; - }; - withCredentials = mkOption { - type = types.bool; - default = false; - description = lib.mdDoc "Enable/disable with credentials headers."; - }; - isDefault = mkOption { - type = types.bool; - default = false; - description = lib.mdDoc "Mark as default datasource. Max one per org."; - }; - jsonData = mkOption { - type = types.nullOr types.attrs; - default = null; - description = lib.mdDoc "Datasource specific configuration."; - }; - secureJsonData = mkOption { - type = types.nullOr types.attrs; - default = null; - description = lib.mdDoc "Datasource specific secure configuration."; - }; - version = mkOption { - type = types.int; - default = 1; - description = lib.mdDoc "Version."; - }; editable = mkOption { type = types.bool; default = false; description = lib.mdDoc "Allow users to edit datasources from the UI."; }; + password = mkOption { + type = types.nullOr types.str; + default = null; + description = lib.mdDoc '' + Database password, if used. Please note that the contents of this option + will end up in a world-readable Nix store. Use the file provider + pointing at a reasonably secured file in the local filesystem + to work around that. Look at the documentation for details: + + ''; + }; + basicAuthPassword = mkOption { + type = types.nullOr types.str; + default = null; + description = lib.mdDoc '' + Basic auth password. Please note that the contents of this option + will end up in a world-readable Nix store. Use the file provider + pointing at a reasonably secured file in the local filesystem + to work around that. Look at the documentation for details: + + ''; + }; + secureJsonData = mkOption { + type = types.nullOr types.attrs; + default = null; + description = lib.mdDoc '' + Datasource specific secure configuration. Please note that the contents of this option + will end up in a world-readable Nix store. Use the file provider + pointing at a reasonably secured file in the local filesystem + to work around that. Look at the documentation for details: + + ''; + }; }; }; @@ -425,10 +402,79 @@ in { enable = mkEnableOption (lib.mdDoc "provision"); datasources = mkOption { - description = lib.mdDoc "Grafana datasources configuration."; + description = lib.mdDoc '' + Deprecated option for Grafana datasource configuration. Use either + `services.grafana.provision.datasources.settings` or + `services.grafana.provision.datasources.path` instead. + ''; default = []; - type = types.listOf grafanaTypes.datasourceConfig; - apply = x: map _filter x; + apply = x: if (builtins.isList x) then map _filter x else x; + type = with types; either (listOf grafanaTypes.datasourceConfig) (submodule { + options.settings = mkOption { + description = lib.mdDoc '' + Grafana datasource configuration in Nix. Can't be used with + `services.grafana.provision.datasources.path` simultaneously. See + + for supported options. + ''; + default = null; + type = types.nullOr (types.submodule { + options = { + apiVersion = mkOption { + description = lib.mdDoc "Config file version."; + default = 1; + type = types.int; + }; + + datasources = mkOption { + description = lib.mdDoc "List of datasources to insert/update."; + default = []; + type = types.listOf grafanaTypes.datasourceConfig; + }; + + deleteDatasources = mkOption { + description = lib.mdDoc "List of datasources that should be deleted from the database."; + default = []; + type = types.listOf (types.submodule { + options.name = mkOption { + description = lib.mdDoc "Name of the datasource to delete."; + type = types.str; + }; + + options.orgId = mkOption { + description = lib.mdDoc "Organization ID of the datasource to delete."; + type = types.int; + }; + }); + }; + }; + }); + example = literalExpression '' + { + apiVersion = 1; + + datasources = [{ + name = "Graphite"; + type = "graphite"; + }]; + + deleteDatasources = [{ + name = "Graphite"; + orgId = 1; + }]; + } + ''; + }; + + options.path = mkOption { + description = lib.mdDoc '' + Path to YAML datasource configuration. Can't be used with + `services.grafana.provision.datasources.settings` simultaneously. + ''; + default = null; + type = types.nullOr types.path; + }; + }); }; @@ -722,11 +768,21 @@ in { cfg.security.adminPassword != opt.security.adminPassword.default ) "Grafana passwords will be stored as plaintext in the Nix store!") (optional ( - any (x: x.password != null || x.basicAuthPassword != null || x.secureJsonData != null) cfg.provision.datasources - ) "Datasource passwords will be stored as plaintext in the Nix store!") + let + checkOpts = opt: any (x: x.password != null || x.basicAuthPassword != null || x.secureJsonData != null) opt; + datasourcesUsed = if (cfg.provision.datasources.settings == null) then [] else cfg.provision.datasources.settings.datasources; + in if (builtins.isList cfg.provision.datasources) then checkOpts cfg.provision.datasources else checkOpts datasourcesUsed + ) "Datasource passwords will be stored as plaintext in the Nix store! Use file provider instead.") (optional ( any (x: x.secure_settings != null) cfg.provision.notifiers ) "Notifier secure settings will be stored as plaintext in the Nix store!") + (optional ( + builtins.isList cfg.provision.datasources + ) '' + Provisioning Grafana datasources with options has been deprecated. + Use `services.grafana.provision.datasources.settings` or + `services.grafana.provision.datasources.path` instead. + '') (optional ( builtins.isList cfg.provision.dashboards ) '' @@ -756,9 +812,17 @@ in { message = "Cannot set both password and passwordFile"; } { - assertion = all + assertion = if (builtins.isList cfg.provision.datasources) then true else cfg.provision.datasources.settings == null || cfg.provision.datasources.path == null; + message = "Cannot set both datasources settings and datasources path"; + } + { + assertion = let + prometheusIsNotDirect = opt: all ({ type, access, ... }: type == "prometheus" -> access != "direct") - cfg.provision.datasources; + opt; + in + if (builtins.isList cfg.provision.datasources) then prometheusIsNotDirect cfg.provision.datasources + else cfg.provision.datasources.settings == null || prometheusIsNotDirect cfg.provision.datasources.settings.datasources; message = "For datasources of type `prometheus`, the `direct` access mode is not supported anymore (since Grafana 9.2.0)"; } { diff --git a/nixos/tests/grafana/default.nix b/nixos/tests/grafana/default.nix index d72fbe4e3f7d..011600b0103d 100644 --- a/nixos/tests/grafana/default.nix +++ b/nixos/tests/grafana/default.nix @@ -5,5 +5,6 @@ { basic = import ./basic.nix { inherit system pkgs; }; + provision-datasources = import ./provision-datasources { inherit system pkgs; }; provision-dashboards = import ./provision-dashboards { inherit system pkgs; }; } diff --git a/nixos/tests/grafana/provision-datasources/default.nix b/nixos/tests/grafana/provision-datasources/default.nix new file mode 100644 index 000000000000..83d5c5607838 --- /dev/null +++ b/nixos/tests/grafana/provision-datasources/default.nix @@ -0,0 +1,95 @@ +args@{ pkgs, ... }: + +(import ../../make-test-python.nix ({ lib, pkgs, ... }: + +let + inherit (lib) mkMerge nameValuePair maintainers; + + baseGrafanaConf = { + services.grafana = { + enable = true; + addr = "localhost"; + analytics.reporting.enable = false; + domain = "localhost"; + security = { + adminUser = "testadmin"; + adminPassword = "snakeoilpwd"; + }; + provision.enable = true; + }; + }; + + extraNodeConfs = { + provisionDatasourceOld = { + services.grafana.provision = { + datasources = [{ + name = "Test Datasource"; + type = "testdata"; + access = "proxy"; + uid = "test_datasource"; + }]; + }; + }; + + provisionDatasourceNix = { + services.grafana.provision = { + datasources.settings = { + apiVersion = 1; + datasources = [{ + name = "Test Datasource"; + type = "testdata"; + access = "proxy"; + uid = "test_datasource"; + }]; + }; + }; + }; + + provisionDatasourceYaml = { + services.grafana.provision.datasources.path = ./provision-datasources.yaml; + }; + }; + + nodes = builtins.listToAttrs (map (provisionType: + nameValuePair provisionType (mkMerge [ + baseGrafanaConf + (extraNodeConfs.${provisionType} or {}) + ])) [ "provisionDatasourceOld" "provisionDatasourceNix" "provisionDatasourceYaml" ]); + +in { + name = "grafana-provision-datasources"; + + meta = with maintainers; { + maintainers = [ kfears willibutz ]; + }; + + inherit nodes; + + testScript = '' + start_all() + + with subtest("Successful datasource provision with Nix (old format)"): + provisionDatasourceOld.wait_for_unit("grafana.service") + provisionDatasourceOld.wait_for_open_port(3000) + provisionDatasourceOld.succeed( + "curl -sSfN -u testadmin:snakeoilpwd http://127.0.0.1:3000/api/datasources/uid/test_datasource | grep Test\ Datasource" + ) + provisionDatasourceOld.shutdown() + + with subtest("Successful datasource provision with Nix (new format)"): + provisionDatasourceNix.wait_for_unit("grafana.service") + provisionDatasourceNix.wait_for_open_port(3000) + provisionDatasourceNix.succeed( + "curl -sSfN -u testadmin:snakeoilpwd http://127.0.0.1:3000/api/datasources/uid/test_datasource | grep Test\ Datasource" + ) + provisionDatasourceNix.shutdown() + + with subtest("Successful datasource provision with YAML"): + provisionDatasourceYaml.wait_for_unit("grafana.service") + provisionDatasourceYaml.wait_for_open_port(3000) + provisionDatasourceYaml.succeed( + "curl -sSfN -u testadmin:snakeoilpwd http://127.0.0.1:3000/api/datasources/uid/test_datasource | grep Test\ Datasource" + ) + provisionDatasourceYaml.shutdown() + ''; +})) args diff --git a/nixos/tests/grafana/provision-datasources/provision-datasources.yaml b/nixos/tests/grafana/provision-datasources/provision-datasources.yaml new file mode 100644 index 000000000000..ccf9481db7f3 --- /dev/null +++ b/nixos/tests/grafana/provision-datasources/provision-datasources.yaml @@ -0,0 +1,7 @@ +apiVersion: 1 + +datasources: + - name: 'Test Datasource' + type: 'testdata' + access: 'proxy' + uid: 'test_datasource'