Merge pull request #68207 from adisbladis/elk-5-removal

elasticsearch5/kibana5/beats5: Remove EOL packages

nixos/doc/manual/release-notes/rl-1909.xml

@@ -428,6 +428,16 @@
      any updates from upstream and depends on outdated Python2-based modules.
     </para>
    </listitem>
+   <listitem>
+    <para>
+     Old unsupported versions (<literal>logstash5</literal>,
+     <literal>kibana5</literal>,
+     <literal>filebeat5</literal>,
+     <literal>heartbeat5</literal>,
+     <literal>metricbeat5</literal>,
+     <literal>packetbeat5</literal>) of the ELK-stack and Elastic beats have been removed.
+    </para>
+   </listitem>
 
   </itemizedlist>
  </section>

nixos/modules/services/logging/logstash.nix

@@ -53,7 +53,7 @@ in
         type = types.package;
         default = pkgs.logstash;
         defaultText = "pkgs.logstash";
-        example = literalExample "pkgs.logstash5";
+        example = literalExample "pkgs.logstash";
         description = "Logstash package to use.";
       };
 

nixos/modules/services/search/kibana.nix

@@ -150,7 +150,7 @@ in {
       description = "Kibana package to use";
       default = pkgs.kibana;
       defaultText = "pkgs.kibana";
-      example = "pkgs.kibana5";
+      example = "pkgs.kibana";
       type = types.package;
     };
 

nixos/tests/elk.nix

@@ -178,12 +178,6 @@ let
     '';
   };
 in mapAttrs mkElkTest {
-  "ELK-5" = {
-    elasticsearch = pkgs.elasticsearch5;
-    logstash      = pkgs.logstash5;
-    kibana        = pkgs.kibana5;
-    journalbeat   = pkgs.journalbeat5;
-  };
   "ELK-6" =
     if enableUnfree
     then {

pkgs/development/tools/misc/kibana/5.x.nix

@@ -1,45 +0,0 @@
-{ stdenv, makeWrapper, fetchurl, elk5Version, nodejs, coreutils, which }:
-
-with stdenv.lib;
-let
-  inherit (builtins) elemAt;
-  archOverrides = {
-    "i686" = "x86";
-  };
-  info = splitString "-" stdenv.hostPlatform.system;
-  arch = (elemAt info 0);
-  elasticArch = archOverrides."${arch}" or arch;
-  plat = elemAt info 1;
-  shas = {
-    "x86_64-linux"  = "0hzr47hyw54b9j4c33n6f6n3pala6kjhyvinfszgikbghyhb7fsa";
-    "i686-linux"    = "0bka4h31cw10ii4pfygc81pwc3wr32pzw3v4k4bi8rnqbk280fmn";
-    "x86_64-darwin" = "0jqc2g89rqkla0alqxr14sh4pccfn514jrwr7mkjivxdapygh1ll";
-  };
-in stdenv.mkDerivation rec {
-  pname = "kibana";
-  version = elk5Version;
-
-  src = fetchurl {
-    url = "https://artifacts.elastic.co/downloads/kibana/${pname}-${version}-${plat}-${elasticArch}.tar.gz";
-    sha256 = shas."${stdenv.hostPlatform.system}" or (throw "Unknown architecture");
-  };
-
-  buildInputs = [ makeWrapper ];
-
-  installPhase = ''
-    mkdir -p $out/libexec/kibana $out/bin
-    mv * $out/libexec/kibana/
-    rm -r $out/libexec/kibana/node
-    makeWrapper $out/libexec/kibana/bin/kibana $out/bin/kibana \
-      --prefix PATH : "${stdenv.lib.makeBinPath [ nodejs coreutils which ]}"
-    sed -i 's@NODE=.*@NODE=${nodejs}/bin/node@' $out/libexec/kibana/bin/kibana
-  '';
-
-  meta = {
-    description = "Visualize logs and time-stamped data";
-    homepage = http://www.elasticsearch.org/overview/kibana;
-    license = licenses.asl20;
-    maintainers = with maintainers; [ offline ];
-    platforms = with platforms; unix;
-  };
-}

pkgs/misc/logging/beats/5.x.nix

@@ -1,42 +0,0 @@
-{ stdenv, fetchFromGitHub, elk5Version, buildGoPackage, libpcap }:
-
-let beat = package : extraArgs : buildGoPackage (rec {
-      name = "${package}-${version}";
-      version = elk5Version;
-
-      src = fetchFromGitHub {
-        owner = "elastic";
-        repo = "beats";
-        rev = "v${version}";
-        sha256 = "0sslcwjdf7zb9xj2c98jid3rb09l96m22k3af48gppimxg1lkh9b";
-      };
-
-      goPackagePath = "github.com/elastic/beats";
-
-      subPackages = [ package ];
-
-      meta = with stdenv.lib; {
-        homepage = https://www.elastic.co/products/beats;
-        license = licenses.asl20;
-        maintainers = with maintainers; [ fadenb basvandijk ];
-        platforms = platforms.linux;
-      };
-    } // extraArgs);
-in {
-  filebeat5   = beat "filebeat"   {meta.description = "Lightweight shipper for logfiles";};
-  heartbeat5  = beat "heartbeat"  {meta.description = "Lightweight shipper for uptime monitoring";};
-  metricbeat5 = beat "metricbeat" {meta.description = "Lightweight shipper for metrics";};
-  packetbeat5 = beat "packetbeat" {
-    buildInputs = [ libpcap ];
-    meta.description = "Network packet analyzer that ships data to Elasticsearch";
-    meta.longDescription = ''
-      Packetbeat is an open source network packet analyzer that ships the
-      data to Elasticsearch.
-
-      Think of it like a distributed real-time Wireshark with a lot more
-      analytics features. The Packetbeat shippers sniff the traffic between
-      your application processes, parse on the fly protocols like HTTP, MySQL,
-      PostgreSQL, Redis or Thrift and correlate the messages into transactions.
-    '';
-  };
-}

pkgs/servers/search/elasticsearch/5.x.nix

@@ -1,44 +0,0 @@
-{ stdenv, fetchurl, elk5Version, makeWrapper, jre_headless
-, utillinux, gnugrep, coreutils }:
-
-with stdenv.lib;
-
-stdenv.mkDerivation rec {
-  version = elk5Version;
-  pname = "elasticsearch";
-
-  src = fetchurl {
-    url = "https://artifacts.elastic.co/downloads/elasticsearch/${pname}-${version}.tar.gz";
-    sha256 = "0zy7awb2cm2fk3c7zc7v8b8pl0jw49awqwpa1jvilmvx6dcml0vb";
-  };
-
-  patches = [ ./es-home-5.x.patch ./es-classpath-5.x.patch ];
-
-  buildInputs = [ makeWrapper jre_headless utillinux ];
-
-  installPhase = ''
-    mkdir -p $out
-    cp -R bin config lib modules plugins $out
-
-    chmod -x $out/bin/*.*
-
-    wrapProgram $out/bin/elasticsearch \
-      --prefix ES_CLASSPATH : "$out/lib/*" \
-      --prefix PATH : "${makeBinPath [ utillinux gnugrep coreutils ]}" \
-      --set JAVA_HOME "${jre_headless}" \
-      --set ES_JVM_OPTIONS "$out/config/jvm.options"
-
-    wrapProgram $out/bin/elasticsearch-plugin \
-      --prefix ES_CLASSPATH : "$out/lib/*" \
-      --set JAVA_HOME "${jre_headless}"
-  '';
-
-  meta = {
-    description = "Open Source, Distributed, RESTful Search Engine";
-    license = licenses.asl20;
-    platforms = platforms.unix;
-    maintainers = [
-      maintainers.apeschar
-    ];
-  };
-}

pkgs/servers/search/elasticsearch/es-classpath-5.x.patch

@@ -1,43 +0,0 @@
-diff -rupN a/bin/elasticsearch b/bin/elasticsearch
---- a/bin/elasticsearch	2017-05-17 10:53:49.444487071 +0200
-+++ b/bin/elasticsearch	2017-05-17 10:55:52.755081523 +0200
-@@ -129,12 +129,7 @@ ES_JAVA_OPTS="$(parse_jvm_options "$ES_J
- # If an include wasn't specified in the environment, then search for one...
- if [ "x$ES_INCLUDE" = "x" ]; then
-     # Locations (in order) to use when searching for an include file.
--    for include in /usr/share/elasticsearch/elasticsearch.in.sh \
--                   /usr/local/share/elasticsearch/elasticsearch.in.sh \
--                   /opt/elasticsearch/elasticsearch.in.sh \
--                   ~/.elasticsearch.in.sh \
--                   "$ES_HOME/bin/elasticsearch.in.sh" \
--                   "`dirname "$0"`"/elasticsearch.in.sh; do
-+    for include in "`dirname "$0"`"/elasticsearch.in.sh; do
-         if [ -r "$include" ]; then
-             . "$include"
-             break
-diff -rupN a/bin/elasticsearch.in.sh b/bin/elasticsearch.in.sh
---- a/bin/elasticsearch.in.sh	2017-04-28 19:41:47.000000000 +0200
-+++ b/bin/elasticsearch.in.sh	2017-05-17 10:56:49.303519788 +0200
-@@ -1,13 +1 @@
- #!/bin/bash
--
--# check in case a user was using this mechanism
--if [ "x$ES_CLASSPATH" != "x" ]; then
--    cat >&2 << EOF
--Error: Don't modify the classpath with ES_CLASSPATH. Best is to add
--additional elements via the plugin mechanism, or if code must really be
--added to the main classpath, add jars to lib/ (unsupported).
--EOF
--    exit 1
--fi
--
--ES_CLASSPATH="$ES_HOME/lib/*"
-diff -rupN a/bin/elasticsearch-plugin b/bin/elasticsearch-plugin
---- a/bin/elasticsearch-plugin	2018-04-13 01:21:55.000000000 +0900
-+++ b/bin/elasticsearch-plugin	2018-06-28 19:08:54.700969245 +0900
-@@ -88,4 +88,4 @@ if [ -e "$CONF_DIR" ]; then
-   path_props=("${path_props[@]}" -Des.path.conf="$CONF_DIR")
- fi
- 
--exec "$JAVA" $ES_JAVA_OPTS -Delasticsearch "${path_props[@]}" -cp "$ES_HOME/lib/*" org.elasticsearch.plugins.PluginCli "${args[@]}"
-+exec "$JAVA" $ES_JAVA_OPTS -Delasticsearch "${path_props[@]}" -cp "$ES_CLASSPATH" org.elasticsearch.plugins.PluginCli "${args[@]}"

pkgs/servers/search/elasticsearch/es-home-5.x.patch

@@ -1,31 +0,0 @@
-diff -rupN a/bin/elasticsearch b/bin/elasticsearch
---- a/bin/elasticsearch	2017-05-17 10:53:42.214686741 +0200
-+++ b/bin/elasticsearch	2017-05-17 10:53:49.444487071 +0200
-@@ -105,7 +105,11 @@ while [ -h "$SCRIPT" ] ; do
- done
- 
- # determine elasticsearch home
--ES_HOME=`dirname "$SCRIPT"`/..
-+
-+if [ -z "$ES_HOME" ]; then
-+    echo "You must set the ES_HOME var" >&2
-+    exit 1
-+fi
- 
- # make ELASTICSEARCH_HOME absolute
- ES_HOME=`cd "$ES_HOME"; pwd`
-diff -rupN a/bin/elasticsearch-plugin b/bin/elasticsearch-plugin
---- a/bin/elasticsearch-plugin	2017-05-17 10:53:42.214686741 +0200
-+++ b/bin/elasticsearch-plugin	2017-05-17 10:53:49.445487044 +0200
-@@ -16,7 +16,10 @@ while [ -h "$SCRIPT" ] ; do
- done
- 
- # determine elasticsearch home
--ES_HOME=`dirname "$SCRIPT"`/..
-+if [ -z "$ES_HOME" ]; then
-+    echo "You must set the ES_HOME var" >&2
-+    exit 1
-+fi
- 
- # make ELASTICSEARCH_HOME absolute
- ES_HOME=`cd "$ES_HOME"; pwd`

pkgs/tools/misc/logstash/5.x.nix

@@ -1,39 +0,0 @@
-{ stdenv, fetchurl, elk5Version, makeWrapper, jre  }:
-
-stdenv.mkDerivation rec {
-  version = elk5Version;
-  pname = "logstash";
-
-  src = fetchurl {
-    url = "https://artifacts.elastic.co/downloads/logstash/${pname}-${version}.tar.gz";
-    sha256 = "0sax9p2bwjdrcvkm1mgvljdjn2qkyjd5i8rzajdn3n98gqin1la0";
-  };
-
-  dontBuild         = true;
-  dontPatchELF      = true;
-  dontStrip         = true;
-  dontPatchShebangs = true;
-
-  buildInputs = [
-    makeWrapper jre
-  ];
-
-  installPhase = ''
-    mkdir -p $out
-    cp -r {Gemfile*,modules,vendor,lib,bin,config,data,logstash-core,logstash-core-plugin-api} $out
-
-    wrapProgram $out/bin/logstash \
-       --set JAVA_HOME "${jre}"
-
-    wrapProgram $out/bin/logstash-plugin \
-       --set JAVA_HOME "${jre}"
-  '';
-
-  meta = with stdenv.lib; {
-    description = "Logstash is a data pipeline that helps you process logs and other event data from a variety of systems";
-    homepage    = https://www.elastic.co/products/logstash;
-    license     = licenses.asl20;
-    platforms   = platforms.unix;
-    maintainers = [ maintainers.wjlroe maintainers.offline ];
-  };
-}

pkgs/tools/system/journalbeat/default.nix

@@ -1,28 +0,0 @@
-{ lib, systemd, buildGoPackage, fetchFromGitHub }:
-
-buildGoPackage rec {
-  pname = "journalbeat";
-  version = "5.6.8";
-
-  src = fetchFromGitHub {
-    owner = "mheese";
-    repo = "journalbeat";
-    rev = "v${version}";
-    sha256 = "1vgpwnwqjc93nvdpcd52748bwl3r371jb55l17bsgdzrmlcyfm8a";
-  };
-
-  goPackagePath = "github.com/mheese/journalbeat";
-
-  buildInputs = [ systemd.dev ];
-
-  postFixup = let libPath = lib.makeLibraryPath [ systemd.lib ]; in ''
-    patchelf --set-rpath ${libPath} "$bin/bin/journalbeat"
-  '';
-
-  meta = with lib; {
-    homepage = https://github.com/mheese/journalbeat;
-    description = "Journalbeat is a log shipper from systemd/journald to Logstash/Elasticsearch";
-    license = licenses.asl20;
-    maintainers = with maintainers; [ mbrgm ];
-  };
-}

pkgs/top-level/all-packages.nix

@@ -1066,18 +1066,6 @@ in
 
   bchunk = callPackage ../tools/cd-dvd/bchunk { };
 
-  inherit (callPackages ../misc/logging/beats/5.x.nix {
-    # XXX: this is failing with Go 1.12. Error is related to cgo, an
-    # update to this package might fix it.
-    buildGoPackage = buildGo111Package;
-  })
-    filebeat5
-    heartbeat5
-    metricbeat5
-    packetbeat5;
-
-  journalbeat5 = callPackage ../tools/system/journalbeat { };
-
   inherit (callPackages ../misc/logging/beats/6.x.nix {
     # XXX: this is failing with Go 1.12. Error is related to cgo, an
     # update to this package might fix it.
@@ -2891,13 +2879,9 @@ in
   evemu = callPackage ../tools/system/evemu { };
 
   # The latest version used by elasticsearch, logstash, kibana and the the beats from elastic.
-  elk5Version = "5.6.16";
   elk6Version = "6.7.2";
   elk7Version = "7.0.1";
 
-  elasticsearch5 = callPackage ../servers/search/elasticsearch/5.x.nix {
-    utillinux = utillinuxMinimal;
-  };
   elasticsearch6 = callPackage ../servers/search/elasticsearch/6.x.nix {
     utillinux = utillinuxMinimal;
   };
@@ -2920,9 +2904,6 @@ in
       elasticsearch = elasticsearch-oss;
     }
   );
-  elasticsearch5Plugins = elasticsearchPlugins.override {
-    elasticsearch = elasticsearch5;
-  };
   elasticsearch6Plugins = elasticsearchPlugins.override {
     elasticsearch = elasticsearch6-oss;
   };
@@ -4215,7 +4196,6 @@ in
 
   keyfuzz = callPackage ../tools/inputmethods/keyfuzz { };
 
-  kibana5 = callPackage ../development/tools/misc/kibana/5.x.nix { };
   kibana6 = callPackage ../development/tools/misc/kibana/6.x.nix { };
   kibana6-oss = callPackage ../development/tools/misc/kibana/6.x.nix {
     enableUnfree = false;
@@ -4317,7 +4297,6 @@ in
 
   lockfileProgs = callPackage ../tools/misc/lockfile-progs { };
 
-  logstash5 = callPackage ../tools/misc/logstash/5.x.nix { };
   logstash6 = callPackage ../tools/misc/logstash/6.x.nix { };
   logstash6-oss = callPackage ../tools/misc/logstash/6.x.nix {
     enableUnfree = false;