dd-agent: Simplify inclusion of additional Datadog core integrations

Refactors the process used to build the Datadog core integrations to
be more easily extensible with integrations other than the ones built
and installed by default.

Documentation has been added in relevant parts of the module to
describe how the process works.

As a high-level overview:

The `datadog-integrations-core` attribute in the top-level package set
now accepts an extra parameter.

This parameter is an attribute set where each key is the name of a
Datadog integration as it appears in Datadog's integrations-core
repository[1], and the value is a function that receives the Python
package set and returns the required dependencies of this integration.

For example:

    datadog-integrations-core {
      ntp = (ps: [ ps.ntplib ]);
    };

This would build the default integrations and, additionally, the `ntp`
integration.

To support passing the modified Python environment to the
datadog-agent itself, the `python` key has been moved inside of the
derivation which means that it will be made overridable.

This relates to NixOS/nixpkgs#40399.

[1]: https://github.com/DataDog/integrations-core

pkgs/tools/networking/dd-agent/6.nix

@@ -1,8 +1,6 @@
 { stdenv, fetchFromGitHub, buildGoPackage, makeWrapper, pythonPackages, pkgconfig }:
 
 let
-  inherit (pythonPackages) python;
-
   # keep this in sync with github.com/DataDog/agent-payload dependency
   payloadVersion = "4.7";
 
@@ -27,6 +25,9 @@ in buildGoPackage rec {
   goDeps = ./deps.nix;
   goPackagePath = "github.com/${owner}/${repo}";
 
+  # Explicitly set this here to allow it to be overridden.
+  python = pythonPackages.python;
+
   nativeBuildInputs = [ pkgconfig makeWrapper ];
   PKG_CONFIG_PATH = "${python}/lib/pkgconfig";
 

pkgs/tools/networking/dd-agent/integrations-core.nix

@@ -1,7 +1,39 @@
-{ pkgs
-, python
-, overrides ? (self: super: {})
-}:
+# The declarations in this file build the Datadog agent's core
+# integrations. These integrations are tracked in a separate
+# repository[1] outside of the agent's primary repository and provide
+# checks for various kinds of services.
+#
+# Not all services are relevant for all users, however. As some of
+# them depend on various tools and Python packages it is nonsensical
+# to build *all* integrations by default.
+#
+# A set of default integrations is defined and built either way.
+# Additional integrations can be specified by overriding
+# `extraIntegrations` in datadog-integrations-core.
+#
+# In practice the syntax for using this with additional integrations
+# is not the most beautiful, but it works. For example to use
+# datadog-agent from the top-level with the `ntp`-integration
+# included, one could say:
+#
+# let
+#   integrationsWithNtp = datadog-integrations-core {
+#     # Extra integrations map from the integration name (as in the
+#     # integrations-core repository) to a function that receives the
+#     # Python package set and returns the required dependencies.g
+#     ntp = (ps: [ ps.ntplib ]);
+#   };
+#
+# in ddAgentWithNtp = datadog-agent.overrideAttrs(_ : {
+#   python = integrationsWithNtp.python;
+# });
+#
+# The NixOS module 'datadog-agent' provides a simplified interface to
+# this. Please see the module itself for more information.
+#
+# [1]: https://github.com/DataDog/integrations-core
+
+{ pkgs, python, extraIntegrations ? {} }:
 
 with pkgs.lib;
 
@@ -14,6 +46,7 @@ let
   };
   version = "git-2018-05-27";
 
+  # Build helper to build a single datadog integration package.
   buildIntegration = { pname, ... }@args: python.pkgs.buildPythonPackage (args // {
     inherit src version;
     name = "datadog-integration-${pname}-${version}";
@@ -27,40 +60,32 @@ let
     doCheck = false;
   });
 
-  packages = (self: {
-    python = python.withPackages (ps: with self; [ disk network postgres nginx mongo ]);
+  # Base package depended on by all other integrations.
+  datadog_checks_base = buildIntegration {
+    pname = "checks-base";
+    sourceRoot = "datadog_checks_base";
+    propagatedBuildInputs = with python.pkgs; [
+      requests protobuf prometheus_client uuid simplejson uptime
+    ];
+  };
 
-    datadog_checks_base = buildIntegration {
-      pname = "checks-base";
-      sourceRoot = "datadog_checks_base";
-      propagatedBuildInputs = with self; with python.pkgs; [ requests protobuf prometheus_client uuid simplejson uptime ];
-    };
+  # Default integrations that should be built:
+  defaultIntegrations = {
+    disk     = (ps: [ ps.psutil ]);
+    mongo    = (ps: [ ps.pymongo ]);
+    network  = (ps: [ ps.psutil ]);
+    nginx    = (ps: []);
+    postgres = (ps: with ps; [ pg8000 psycopg2 ]);
+  };
 
-    disk = buildIntegration {
-      pname = "disk";
-      propagatedBuildInputs = with self; with python.pkgs; [ datadog_checks_base psutil ];
-    };
+  # All integrations (default + extra):
+  integrations = defaultIntegrations // extraIntegrations;
+  builtIntegrations = mapAttrs (pname: fdeps: buildIntegration {
+    inherit pname;
+    propagatedBuildInputs = (fdeps python.pkgs) ++ [ datadog_checks_base ];
+  }) integrations;
 
-    network = buildIntegration {
-      pname = "network";
-      propagatedBuildInputs = with self; with python.pkgs; [ datadog_checks_base psutil ];
-    };
-
-    postgres = buildIntegration {
-      pname = "postgres";
-      propagatedBuildInputs = with self; with python.pkgs; [ datadog_checks_base pg8000 psycopg2 ];
-    };
-
-    nginx = buildIntegration {
-      pname = "nginx";
-      propagatedBuildInputs = with self; with python.pkgs; [ datadog_checks_base ];
-    };
-
-    mongo = buildIntegration {
-      pname = "mongo";
-      propagatedBuildInputs = with self; with python.pkgs; [ datadog_checks_base pymongo ];
-    };
-
-  });
-
-in fix' (extends overrides packages)
+in builtIntegrations // {
+  inherit datadog_checks_base;
+  python = python.withPackages (_: (attrValues builtIntegrations));
+}

pkgs/top-level/all-packages.nix

@@ -15674,10 +15674,11 @@ with pkgs;
 
   dd-agent = callPackage ../tools/networking/dd-agent/5.nix { };
   datadog-agent = callPackage ../tools/networking/dd-agent/6.nix {
-    pythonPackages = datadog-integrations-core;
+    pythonPackages = datadog-integrations-core {};
   };
-  datadog-integrations-core = callPackage ../tools/networking/dd-agent/integrations-core.nix {
+  datadog-integrations-core = extras: callPackage ../tools/networking/dd-agent/integrations-core.nix {
     python = python27;
+    extraIntegrations = extras;
   };
 
   ddgr = callPackage ../applications/misc/ddgr { };