Merge pull request #120009 from DeterminateSystems/scancode

python3.pkgs.scancode-toolkit: init at 21.3.31

pkgs/development/python-modules/banal/default.nix

@@ -0,0 +1,27 @@
+{ lib
+, fetchPypi
+, buildPythonPackage
+}:
+buildPythonPackage rec {
+  pname = "banal";
+  version = "1.0.6";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "2fe02c9305f53168441948f4a03dfbfa2eacc73db30db4a93309083cb0e250a5";
+  };
+
+  # no tests
+  doCheck = false;
+
+  pythonImportsCheck = [
+    "banal"
+  ];
+
+  meta = with lib; {
+    description = "Commons of banal micro-functions for Python";
+    homepage = "https://github.com/pudo/banal";
+    license = licenses.mit;
+    maintainers = teams.determinatesystems.members;
+  };
+}

pkgs/development/python-modules/commoncode/default.nix

@@ -0,0 +1,55 @@
+{ lib
+, fetchPypi
+, buildPythonPackage
+, setuptools-scm
+, click
+, requests
+, attrs
+, intbitset
+, saneyaml
+, text-unidecode
+, beautifulsoup4
+, pytestCheckHook
+, pytest-xdist
+}:
+buildPythonPackage rec {
+  pname = "commoncode";
+  version = "21.1.21";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "6e2daa34fac2d91307b23d9df5f01a6168fdffb12bf5d161bd6776bade29b479";
+  };
+
+  dontConfigure = true;
+
+  nativeBuildInputs = [
+    setuptools-scm
+  ];
+
+  propagatedBuildInputs = [
+    click
+    requests
+    attrs
+    intbitset
+    saneyaml
+    text-unidecode
+    beautifulsoup4
+  ];
+
+  checkInputs = [
+    pytestCheckHook
+    pytest-xdist
+  ];
+
+  pythonImportsCheck = [
+    "commoncode"
+  ];
+
+  meta = with lib; {
+    description = "A set of common utilities, originally split from ScanCode";
+    homepage = "https://github.com/nexB/commoncode";
+    license = licenses.asl20;
+    maintainers = teams.determinatesystems.members;
+  };
+}

pkgs/development/python-modules/debian-inspector/default.nix

@@ -0,0 +1,53 @@
+{ lib
+, buildPythonPackage
+, fetchPypi
+, pythonAtLeast
+, fetchpatch
+, chardet
+, attrs
+, commoncode
+, pytestCheckHook
+}:
+buildPythonPackage rec {
+  pname = "debian-inspector";
+  version = "0.9.10";
+
+  src = fetchPypi {
+    pname = "debian_inspector";
+    inherit version;
+    sha256 = "fd29a02b925a4de0d7bb00c29bb05f19715a304bc10ef7b9ad06a93893dc3a8c";
+  };
+
+  patches = lib.optionals (pythonAtLeast "3.9") [
+    # https://github.com/nexB/debian-inspector/pull/15
+    # fixes tests on Python 3.9
+    (fetchpatch {
+      name = "drop-encoding-argument.patch";
+      url = "https://github.com/nexB/debian-inspector/commit/ff991cdb788671ca9b81f1602b70a439248fd1aa.patch";
+      sha256 = "bm3k7vb9+Rm6+YicQEeDOOUVut8xpDaNavG+t2oLZkI=";
+    })
+  ];
+
+  dontConfigure = true;
+
+  propagatedBuildInputs = [
+    chardet
+    attrs
+    commoncode
+  ];
+
+  checkInputs = [
+    pytestCheckHook
+  ];
+
+  pythonImportsCheck = [
+    "debian_inspector"
+  ];
+
+  meta = with lib; {
+    description = "Utilities to parse Debian package, copyright and control files";
+    homepage = "https://github.com/nexB/debian-inspector";
+    license = with licenses; [ asl20 bsd3 mit ];
+    maintainers = teams.determinatesystems.members;
+  };
+}

pkgs/development/python-modules/dparse/default.nix

@@ -1,4 +1,12 @@
-{ lib, buildPythonPackage, fetchPypi, isPy27, pytest, toml, pyyaml }:
+{ lib
+, buildPythonPackage
+, fetchPypi
+, isPy27
+, toml
+, pyyaml
+, packaging
+, pytestCheckHook
+}:
 
 buildPythonPackage rec {
   pname = "dparse";
@@ -10,13 +18,24 @@ buildPythonPackage rec {
     sha256 = "a1b5f169102e1c894f9a7d5ccf6f9402a836a5d24be80a986c7ce9eaed78f367";
   };
 
-  propagatedBuildInputs = [ toml pyyaml ];
+  propagatedBuildInputs = [
+    toml
+    pyyaml
+    packaging
+  ];
 
-  checkInputs = [ pytest ];
+  checkInputs = [
+    pytestCheckHook
+  ];
+
+  disabledTests = [
+    # requires unpackaged dependency pipenv
+    "test_update_pipfile"
+  ];
 
   meta = with lib; {
     description = "A parser for Python dependency files";
-    homepage = "https://github.com/pyupio/safety";
+    homepage = "https://github.com/pyupio/dparse";
     license = licenses.mit;
     maintainers = with maintainers; [ thomasdesr ];
   };

pkgs/development/python-modules/extractcode/7z.nix

@@ -0,0 +1,48 @@
+{ lib
+, fetchFromGitHub
+, buildPythonPackage
+, plugincode
+, p7zip
+}:
+buildPythonPackage rec {
+  pname = "extractcode-7z";
+  version = "21.4.4";
+
+  src = fetchFromGitHub {
+    owner = "nexB";
+    repo = "scancode-plugins";
+    rev = "v${version}";
+    sha256 = "xnUGDMS34iMVMGo/nZwRarGzzbj3X4Rt+YHvvKpmy6A=";
+  };
+
+  sourceRoot = "source/builtins/extractcode_7z-linux";
+
+  propagatedBuildInputs = [
+    plugincode
+  ];
+
+  preBuild = ''
+    pushd src/extractcode_7z/bin
+
+    rm 7z 7z.so
+    ln -s ${p7zip}/bin/7z 7z
+    ln -s ${p7zip}/lib/p7zip/7z.so 7z.so
+
+    popd
+  '';
+
+  # no tests
+  doCheck = false;
+
+  pythonImportsCheck = [
+    "extractcode_7z"
+  ];
+
+  meta = with lib; {
+    description = "A ScanCode Toolkit plugin to provide pre-built binary libraries and utilities and their locations";
+    homepage = "https://github.com/nexB/scancode-plugins/tree/main/builtins/extractcode_7z-linux";
+    license = with licenses; [ asl20 lgpl21 ];
+    maintainers = teams.determinatesystems.members;
+    platforms = platforms.linux;
+  };
+}

pkgs/development/python-modules/extractcode/default.nix

@@ -0,0 +1,60 @@
+{ lib
+, fetchPypi
+, buildPythonPackage
+, setuptools-scm
+, typecode
+, patch
+, extractcode-libarchive
+, extractcode-7z
+, pytestCheckHook
+, pytest-xdist
+}:
+buildPythonPackage rec {
+  pname = "extractcode";
+  version = "21.2.24";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "f91638dbf523b80df90ac184c25d5cd1ea24cac53f67a6bb7d7b389867e0744b";
+  };
+
+  dontConfigure = true;
+
+  nativeBuildInputs = [
+    setuptools-scm
+  ];
+
+  propagatedBuildInputs = [
+    typecode
+    patch
+    extractcode-libarchive
+    extractcode-7z
+  ];
+
+  checkInputs = [
+    pytestCheckHook
+    pytest-xdist
+  ];
+
+  # cli test tests the cli which we can't do until after install
+  disabledTestPaths = [
+    "tests/test_extractcode_cli.py"
+  ];
+
+  # test_uncompress_* wants to use a binary to extract instead of the provided library
+  disabledTests = [
+    "test_uncompress_lz4_basic"
+    "test_extract_tarlz4_basic"
+  ];
+
+  pythonImportsCheck = [
+    "extractcode"
+  ];
+
+  meta = with lib; {
+    description = "A mostly universal archive extractor using z7zip, libarchve, other libraries and the Python standard library for reliable archive extraction";
+    homepage = "https://github.com/nexB/extractcode";
+    license = licenses.asl20;
+    maintainers = teams.determinatesystems.members;
+  };
+}

pkgs/development/python-modules/extractcode/libarchive.nix

@@ -0,0 +1,62 @@
+{ lib
+, fetchFromGitHub
+, buildPythonPackage
+, libarchive
+, libb2
+, bzip2
+, expat
+, lz4
+, lzma
+, zlib
+, zstd
+, plugincode
+, pytestCheckHook
+}:
+buildPythonPackage rec {
+  pname = "extractcode-libarchive";
+  version = "21.4.4";
+
+  src = fetchFromGitHub {
+    owner = "nexB";
+    repo = "scancode-plugins";
+    rev = "v${version}";
+    sha256 = "xnUGDMS34iMVMGo/nZwRarGzzbj3X4Rt+YHvvKpmy6A=";
+  };
+
+  sourceRoot = "source/builtins/extractcode_libarchive-linux";
+
+  preBuild = ''
+    pushd src/extractcode_libarchive/lib
+
+    rm *.so *.so.*
+    ln -s ${lib.getLib libarchive}/lib/libarchive.so libarchive.so
+    ln -s ${lib.getLib libb2}/lib/libb2.so libb2-la3511.so.1
+    ln -s ${lib.getLib bzip2}/lib/libbz2.so libbz2-la3511.so.1.0
+    ln -s ${lib.getLib expat}/lib/libexpat.so libexpat-la3511.so.1
+    ln -s ${lib.getLib lz4}/lib/liblz4.so liblz4-la3511.so.1
+    ln -s ${lib.getLib lzma}/lib/liblzma.so liblzma-la3511.so.5
+    ln -s ${lib.getLib zlib}/lib/libz.so libz-la3511.so.1
+    ln -s ${lib.getLib zstd}/lib/libzstd.so libzstd-la3511.so.1
+
+    popd
+  '';
+
+  propagatedBuildInputs = [
+    plugincode
+  ];
+
+  # no tests
+  doCheck = false;
+
+  pythonImportsCheck = [
+    "extractcode_libarchive"
+  ];
+
+  meta = with lib; {
+    description = "A ScanCode Toolkit plugin to provide pre-built binary libraries and utilities and their locations";
+    homepage = "https://github.com/nexB/scancode-plugins/tree/main/builtins/extractcode_libarchive-linux";
+    license = with licenses; [ asl20 bsd2 ];
+    maintainers = teams.determinatesystems.members;
+    platforms = platforms.linux;
+  };
+}

pkgs/development/python-modules/fingerprints/default.nix

@@ -0,0 +1,42 @@
+{ lib
+, fetchPypi
+, buildPythonPackage
+, normality
+, mypy
+, coverage
+, nose
+}:
+buildPythonPackage rec {
+  pname = "fingerprints";
+  version = "1.0.3";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "cafd5f92b5b91e4ce34af2b954da9c05b448a4778947785abb19a14f363352d0";
+  };
+
+  propagatedBuildInputs = [
+    normality
+  ];
+
+  checkInputs = [
+    mypy
+    coverage
+    nose
+  ];
+
+  checkPhase = ''
+    nosetests
+  '';
+
+  pythonImportsCheck = [
+    "fingerprints"
+  ];
+
+  meta = with lib; {
+    description = "A library to generate entity fingerprints";
+    homepage = "https://github.com/alephdata/fingerprints";
+    license = licenses.mit;
+    maintainers = teams.determinatesystems.members;
+  };
+}

pkgs/development/python-modules/gemfileparser/default.nix

@@ -0,0 +1,29 @@
+{ lib
+, fetchPypi
+, buildPythonPackage
+, pytestCheckHook
+}:
+buildPythonPackage rec {
+  pname = "gemfileparser";
+  version = "0.8.0";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "839592e49ea3fd985cec003ef58f8e77009a69ed7644a0c0acc94cf6dd9b8d6e";
+  };
+
+  checkInputs = [
+    pytestCheckHook
+  ];
+
+  pythonImportsCheck = [
+    "gemfileparser"
+  ];
+
+  meta = with lib; {
+    description = "A library to parse Ruby Gemfile, .gemspec and Cocoapod .podspec file using Python";
+    homepage = "https://github.com/gemfileparser/gemfileparser";
+    license = with licenses; [ gpl3Plus /* or */ mit ];
+    maintainers = teams.determinatesystems.members;
+  };
+}

pkgs/development/python-modules/intbitset/default.nix

@@ -0,0 +1,44 @@
+{ lib
+, fetchPypi
+, buildPythonPackage
+, six
+, nose
+}:
+buildPythonPackage rec {
+  pname = "intbitset";
+  version = "2.4.1";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "44bca80b8cc702d5a56f0686f2bb5e028ab4d0c2c1761941589d46b7fa2c701c";
+  };
+
+  patches = [
+    # fixes compilation on aarch64 and determinism (uses -march=core2 and
+    # -mtune=native)
+    ./remove-impure-tuning.patch
+  ];
+
+  propagatedBuildInputs = [
+    six
+  ];
+
+  checkInputs = [
+    nose
+  ];
+
+  checkPhase = ''
+    nosetests
+  '';
+
+  pythonImportsCheck = [
+    "intbitset"
+  ];
+
+  meta = with lib; {
+    description = "C-based extension implementing fast integer bit sets";
+    homepage = "https://github.com/inveniosoftware/intbitset";
+    license = licenses.lgpl3Only;
+    maintainers = teams.determinatesystems.members;
+  };
+}

pkgs/development/python-modules/intbitset/remove-impure-tuning.patch

@@ -0,0 +1,24 @@
+From 2ea60bdf4d7b0344fc6ff5c97c675842fedccfa8 Mon Sep 17 00:00:00 2001
+From: Cole Helbling <cole.e.helbling@outlook.com>
+Date: Fri, 23 Apr 2021 09:02:22 -0700
+Subject: [PATCH] setup.py: remove impure tuning
+
+---
+ setup.py | 1 -
+ 1 file changed, 1 deletion(-)
+
+diff --git a/setup.py b/setup.py
+index 7840022..3922aa5 100644
+--- a/setup.py
++++ b/setup.py
+@@ -48,7 +48,6 @@ setup(
+     ext_modules=[
+         Extension("intbitset",
+                   ["intbitset/intbitset.c", "intbitset/intbitset_impl.c"],
+-                  extra_compile_args=['-O3', '-march=core2', '-mtune=native']
+                   # For debug -> '-ftree-vectorizer-verbose=2'
+                   )
+     ],
+-- 
+2.30.1
+

pkgs/development/python-modules/normality/default.nix

@@ -0,0 +1,42 @@
+{ lib
+, fetchFromGitHub
+, buildPythonPackage
+, text-unidecode
+, chardet
+, banal
+, PyICU
+, pytestCheckHook
+}:
+buildPythonPackage rec {
+  pname = "normality";
+  version = "2.1.3";
+
+  src = fetchFromGitHub {
+    owner = "pudo";
+    repo = "normality";
+    rev = version;
+    sha256 = "WvpMs02vBGnCSPkxo6r6g4Di2fKkUr2SsBflTBxlhkU=";
+  };
+
+  propagatedBuildInputs = [
+    text-unidecode
+    chardet
+    banal
+    PyICU
+  ];
+
+  checkInputs = [
+    pytestCheckHook
+  ];
+
+  pythonImportsCheck = [
+    "normality"
+  ];
+
+  meta = with lib; {
+    description = "Micro-library to normalize text strings";
+    homepage = "https://github.com/pudo/normality";
+    license = licenses.mit;
+    maintainers = teams.determinatesystems.members;
+  };
+}

pkgs/development/python-modules/plugincode/default.nix

@@ -0,0 +1,47 @@
+{ lib
+, fetchPypi
+, buildPythonPackage
+, setuptools-scm
+, click
+, commoncode
+, pluggy
+, pytestCheckHook
+, pytest-xdist
+}:
+buildPythonPackage rec {
+  pname = "plugincode";
+  version = "21.1.21";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "97b5a2c96f0365c80240be103ecd86411c68b11a16f137913cbea9129c54907a";
+  };
+
+  dontConfigure = true;
+
+  nativeBuildInputs = [
+    setuptools-scm
+  ];
+
+  propagatedBuildInputs = [
+    click
+    commoncode
+    pluggy
+  ];
+
+  checkInputs = [
+    pytestCheckHook
+    pytest-xdist
+  ];
+
+  pythonImportsCheck = [
+    "plugincode"
+  ];
+
+  meta = with lib; {
+    description = "A library that provides plugin functionality for ScanCode toolkit";
+    homepage = "https://github.com/nexB/plugincode";
+    license = licenses.asl20;
+    maintainers = teams.determinatesystems.members;
+  };
+}

pkgs/development/python-modules/pymaven-patch/default.nix

@@ -0,0 +1,44 @@
+{ lib
+, fetchPypi
+, buildPythonPackage
+, pbr
+, requests
+, six
+, lxml
+, pytestCheckHook
+, pytest-cov
+, mock
+}:
+buildPythonPackage rec {
+  pname = "pymaven-patch";
+  version = "0.3.0";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "d55b29bd4aeef3510904a12885eb6856b5bd48f3e29925a123461429f9ad85c0";
+  };
+
+  propagatedBuildInputs = [
+    pbr
+    requests
+    six
+    lxml
+  ];
+
+  checkInputs = [
+    pytestCheckHook
+    pytest-cov
+    mock
+  ];
+
+  pythonImportsCheck = [
+    "pymaven"
+  ];
+
+  meta = with lib; {
+    description = "Python access to maven";
+    homepage = "https://github.com/nexB/pymaven";
+    license = licenses.asl20;
+    maintainers = teams.determinatesystems.members;
+  };
+}

pkgs/development/python-modules/saneyaml/default.nix

@@ -0,0 +1,41 @@
+{ lib
+, fetchPypi
+, buildPythonPackage
+, setuptools-scm
+, pyyaml
+, pytestCheckHook
+}:
+buildPythonPackage rec {
+  pname = "saneyaml";
+  version = "0.5.2";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "d6074f1959041342ab41d74a6f904720ffbcf63c94467858e0e22e17e3c43d41";
+  };
+
+  dontConfigure = true;
+
+  nativeBuildInputs = [
+    setuptools-scm
+  ];
+
+  propagatedBuildInputs = [
+    pyyaml
+  ];
+
+  checkInputs = [
+    pytestCheckHook
+  ];
+
+  pythonImportsCheck = [
+    "saneyaml"
+  ];
+
+  meta = with lib; {
+    description = "A PyYaml wrapper with sane behaviour to read and write readable YAML safely";
+    homepage = "https://github.com/nexB/saneyaml";
+    license = licenses.asl20;
+    maintainers = teams.determinatesystems.members;
+  };
+}

pkgs/development/python-modules/scancode-toolkit/default.nix

@@ -0,0 +1,122 @@
+{ lib
+, fetchPypi
+, buildPythonPackage
+, isPy3k
+, markupsafe
+, click
+, typecode
+, gemfileparser
+, pefile
+, fingerprints
+, spdx-tools
+, fasteners
+, pycryptodome
+, urlpy
+, dparse
+, jaraco_functools
+, pkginfo
+, debian-inspector
+, extractcode
+, ftfy
+, pyahocorasick
+, colorama
+, jsonstreams
+, packageurl-python
+, pymaven-patch
+, nltk
+, pygments
+, bitarray
+, jinja2
+, javaproperties
+, boolean-py
+, license-expression
+, extractcode-7z
+, extractcode-libarchive
+, typecode-libmagic
+, pytestCheckHook
+}:
+buildPythonPackage rec {
+  pname = "scancode-toolkit";
+  version = "21.3.31";
+  disabled = !isPy3k;
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "7e0301031a302dedbb4304a91249534f3d036f84a119170b8a9fe70bd57cff95";
+  };
+
+  dontConfigure = true;
+
+  # https://github.com/nexB/scancode-toolkit/issues/2501
+  # * dparse2 is a "Temp fork for Python 2 support", but pdfminer requires
+  # Python 3, so it's "fine" to leave dparse2 unpackaged and use the "normal"
+  # version
+  # * ftfy was pinned for similar reasons (to support Python 2), but rather than
+  # packaging an older version, I figured it would be better to remove the
+  # erroneous (at least for our usage) version bound
+  # * bitarray's version bound appears to be unnecessary for similar reasons
+  postPatch = ''
+    substituteInPlace setup.cfg \
+      --replace "dparse2" "dparse" \
+      --replace "ftfy <  5.0.0" "ftfy" \
+      --replace "bitarray >= 0.8.1, < 1.0.0" "bitarray"
+  '';
+
+  propagatedBuildInputs = [
+    markupsafe
+    click
+    typecode
+    gemfileparser
+    pefile
+    fingerprints
+    spdx-tools
+    fasteners
+    pycryptodome
+    urlpy
+    dparse
+    jaraco_functools
+    pkginfo
+    debian-inspector
+    extractcode
+    ftfy
+    pyahocorasick
+    colorama
+    jsonstreams
+    packageurl-python
+    pymaven-patch
+    nltk
+    pygments
+    bitarray
+    jinja2
+    javaproperties
+    boolean-py
+    license-expression
+    extractcode-7z
+    extractcode-libarchive
+    typecode-libmagic
+  ];
+
+  checkInputs = [
+    pytestCheckHook
+  ];
+
+  # Importing scancode needs a writeable home, and preCheck happens in between
+  # pythonImportsCheckPhase and pytestCheckPhase.
+  postInstall = ''
+    export HOME=$(mktemp -d)
+  '';
+
+  pythonImportsCheck = [
+    "scancode"
+  ];
+
+  # takes a long time and doesn't appear to do anything
+  dontStrip = true;
+
+  meta = with lib; {
+    description = "A tool to scan code for license, copyright, package and their documented dependencies and other interesting facts";
+    homepage = "https://github.com/nexB/scancode-toolkit";
+    license = with licenses; [ asl20 cc-by-40 ];
+    maintainers = teams.determinatesystems.members;
+  };
+}

pkgs/development/python-modules/spdx-tools/default.nix

@@ -0,0 +1,54 @@
+{ lib
+, buildPythonPackage
+, fetchPypi
+, fetchpatch
+, six
+, pyyaml
+, rdflib
+, ply
+, xmltodict
+, pytestCheckHook
+, pythonAtLeast
+}:
+buildPythonPackage rec {
+  pname = "spdx-tools";
+  version = "0.6.1";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "9a1aaae051771e865705dd2fd374c3f73d0ad595c1056548466997551cbd7a81";
+  };
+
+  patches = lib.optionals (pythonAtLeast "3.9") [
+    # https://github.com/spdx/tools-python/pull/159
+    # fixes tests on Python 3.9
+    (fetchpatch {
+      name = "drop-encoding-argument.patch";
+      url = "https://github.com/spdx/tools-python/commit/6c8b9a852f8a787122c0e2492126ee8aa52acff0.patch";
+      sha256 = "RhvLhexsQRjqYqJg10SAM53RsOW+R93G+mns8C9g5E8=";
+    })
+  ];
+
+  propagatedBuildInputs = [
+    six
+    pyyaml
+    rdflib
+    ply
+    xmltodict
+  ];
+
+  checkInputs = [
+    pytestCheckHook
+  ];
+
+  pythonImportsCheck = [
+    "spdx"
+  ];
+
+  meta = with lib; {
+    description = "SPDX parser and tools";
+    homepage = "https://github.com/spdx/tools-python";
+    license = licenses.asl20;
+    maintainers = teams.determinatesystems.members;
+  };
+}

pkgs/development/python-modules/typecode/default.nix

@@ -0,0 +1,53 @@
+{ lib
+, fetchPypi
+, buildPythonPackage
+, setuptools-scm
+, attrs
+, pdfminer
+, commoncode
+, plugincode
+, binaryornot
+, typecode-libmagic
+, pytestCheckHook
+, pytest-xdist
+}:
+buildPythonPackage rec {
+  pname = "typecode";
+  version = "21.2.24";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "eaac8aee0b9c6142ad44671252ba00748202d218347d1c0451ce6cd76561e01b";
+  };
+
+  dontConfigure = true;
+
+  nativeBuildInputs = [
+    setuptools-scm
+  ];
+
+  propagatedBuildInputs = [
+    attrs
+    pdfminer
+    commoncode
+    plugincode
+    binaryornot
+    typecode-libmagic
+  ];
+
+  checkInputs = [
+    pytestCheckHook
+    pytest-xdist
+  ];
+
+  pythonImportsCheck = [
+    "typecode"
+  ];
+
+  meta = with lib; {
+    description = "Comprehensive filetype and mimetype detection using libmagic and Pygments";
+    homepage = "https://github.com/nexB/typecode";
+    license = licenses.asl20;
+    maintainers = teams.determinatesystems.members;
+  };
+}

pkgs/development/python-modules/typecode/libmagic.nix

@@ -0,0 +1,51 @@
+{ lib
+, fetchFromGitHub
+, buildPythonPackage
+, plugincode
+, file
+, zlib
+, pytest
+}:
+buildPythonPackage rec {
+  pname = "typecode-libmagic";
+  version = "21.4.4";
+
+  src = fetchFromGitHub {
+    owner = "nexB";
+    repo = "scancode-plugins";
+    rev = "v${version}";
+    sha256 = "xnUGDMS34iMVMGo/nZwRarGzzbj3X4Rt+YHvvKpmy6A=";
+  };
+
+  sourceRoot = "source/builtins/typecode_libmagic-linux";
+
+  propagatedBuildInputs = [
+    plugincode
+  ];
+
+  preBuild = ''
+    pushd src/typecode_libmagic
+
+    rm data/magic.mgc lib/libmagic.so lib/libz-lm539.so.1
+    ln -s ${file}/share/misc/magic.mgc data/magic.mgc
+    ln -s ${file}/lib/libmagic.so lib/libmagic.so
+    ln -s ${zlib}/lib/libz.so lib/libz-lm539.so.1
+
+    popd
+  '';
+
+  # no tests
+  doCheck = false;
+
+  pythonImportsCheck = [
+    "typecode_libmagic"
+  ];
+
+  meta = with lib; {
+    description = "A ScanCode Toolkit plugin to provide pre-built binary libraries and utilities and their locations";
+    homepage = "https://github.com/nexB/scancode-plugins/tree/main/builtins/typecode_libmagic-linux";
+    license = licenses.asl20;
+    maintainers = teams.determinatesystems.members;
+    platforms = platforms.linux;
+  };
+}

pkgs/development/python-modules/urlpy/default.nix

@@ -0,0 +1,44 @@
+{ lib
+, fetchFromGitHub
+, buildPythonPackage
+, publicsuffix2
+, pytestCheckHook
+, pythonAtLeast
+}:
+buildPythonPackage rec {
+  pname = "urlpy";
+  version = "0.5.0";
+
+  src = fetchFromGitHub {
+    owner = "nexB";
+    repo = "urlpy";
+    rev = "v${version}";
+    sha256 = "962jLyx+/GS8wrDPzG2ONnHvtUG5Pqe6l1Z5ml63Cmg=";
+  };
+
+  dontConfigure = true;
+
+  propagatedBuildInputs = [
+    publicsuffix2
+  ];
+
+  checkInputs = [
+    pytestCheckHook
+  ];
+
+  disabledTests = lib.optionals (pythonAtLeast "3.9") [
+    # Fails with "AssertionError: assert 'unknown' == ''"
+    "test_unknown_protocol"
+  ];
+
+  pythonImportsCheck = [
+    "urlpy"
+  ];
+
+  meta = with lib; {
+    description = "Simple URL parsing, canonicalization and equivalence";
+    homepage = "https://github.com/nexB/urlpy";
+    license = licenses.mit;
+    maintainers = teams.determinatesystems.members;
+  };
+}

pkgs/top-level/python-packages.nix

@@ -941,6 +941,8 @@ in {
 
   bacpypes = callPackage ../development/python-modules/bacpypes { };
 
+  banal = callPackage ../development/python-modules/banal { };
+
   bandit = callPackage ../development/python-modules/bandit { };
 
   bap = callPackage ../development/python-modules/bap {
@@ -1513,6 +1515,8 @@ in {
 
   commandparse = callPackage ../development/python-modules/commandparse { };
 
+  commoncode = callPackage ../development/python-modules/commoncode { };
+
   CommonMark = callPackage ../development/python-modules/commonmark { };
 
   compiledb = callPackage ../development/python-modules/compiledb { };
@@ -1777,6 +1781,8 @@ in {
 
   debian = callPackage ../development/python-modules/debian { };
 
+  debian-inspector = callPackage ../development/python-modules/debian-inspector { };
+
   debts = callPackage ../development/python-modules/debts { };
 
   debugpy = callPackage ../development/python-modules/debugpy { };
@@ -2295,6 +2301,24 @@ in {
 
   exrex = callPackage ../development/python-modules/exrex { };
 
+  extractcode = callPackage ../development/python-modules/extractcode { };
+
+  extractcode-7z = callPackage ../development/python-modules/extractcode/7z.nix {
+    inherit (pkgs) p7zip;
+  };
+
+  extractcode-libarchive = callPackage ../development/python-modules/extractcode/libarchive.nix {
+    inherit (pkgs)
+      libarchive
+      libb2
+      bzip2
+      expat
+      lz4
+      lzma
+      zlib
+      zstd;
+  };
+
   extras = callPackage ../development/python-modules/extras { };
 
   eyeD3 = callPackage ../development/python-modules/eyed3 { };
@@ -2405,6 +2429,8 @@ in {
 
   finalfusion = callPackage ../development/python-modules/finalfusion { };
 
+  fingerprints = callPackage ../development/python-modules/fingerprints { };
+
   fints = callPackage ../development/python-modules/fints { };
 
   fiona = callPackage ../development/python-modules/fiona { };
@@ -2690,6 +2716,8 @@ in {
 
   geeknote = callPackage ../development/python-modules/geeknote { };
 
+  gemfileparser = callPackage ../development/python-modules/gemfileparser { };
+
   genanki = callPackage ../development/python-modules/genanki { };
 
   genome-collector = callPackage ../development/python-modules/genome-collector { };
@@ -3395,6 +3423,8 @@ in {
 
   intake = callPackage ../development/python-modules/intake { };
 
+  intbitset = callPackage ../development/python-modules/intbitset { };
+
   intelhex = callPackage ../development/python-modules/intelhex { };
 
   internetarchive = callPackage ../development/python-modules/internetarchive { };
@@ -4741,6 +4771,8 @@ in {
 
   noiseprotocol = callPackage ../development/python-modules/noiseprotocol { };
 
+  normality = callPackage ../development/python-modules/normality { };
+
   nose2 = callPackage ../development/python-modules/nose2 { };
 
   nose = callPackage ../development/python-modules/nose { };
@@ -5327,6 +5359,8 @@ in {
 
   pluggy = callPackage ../development/python-modules/pluggy { };
 
+  plugincode = callPackage ../development/python-modules/plugincode { };
+
   pluginbase = callPackage ../development/python-modules/pluginbase { };
 
   plugnplay = callPackage ../development/python-modules/plugnplay { };
@@ -6096,6 +6130,8 @@ in {
 
   pymatgen-lammps = callPackage ../development/python-modules/pymatgen-lammps { };
 
+  pymaven-patch = callPackage ../development/python-modules/pymaven-patch { };
+
   pymavlink = callPackage ../development/python-modules/pymavlink { };
 
   pymazda = callPackage ../development/python-modules/pymazda { };
@@ -7616,6 +7652,8 @@ in {
     inherit (pkgs) sane-backends;
   };
 
+  saneyaml = callPackage ../development/python-modules/saneyaml { };
+
   sampledata = callPackage ../development/python-modules/sampledata { };
 
   samplerate = callPackage ../development/python-modules/samplerate { };
@@ -7638,6 +7676,8 @@ in {
 
   scales = callPackage ../development/python-modules/scales { };
 
+  scancode-toolkit = callPackage ../development/python-modules/scancode-toolkit { };
+
   scandir = callPackage ../development/python-modules/scandir { };
 
   scapy = callPackage ../development/python-modules/scapy { };
@@ -8040,6 +8080,8 @@ in {
 
   sparse = callPackage ../development/python-modules/sparse { };
 
+  spdx-tools = callPackage ../development/python-modules/spdx-tools { };
+
   speaklater = callPackage ../development/python-modules/speaklater { };
 
   spectral-cube = callPackage ../development/python-modules/spectral-cube { };
@@ -8716,6 +8758,12 @@ in {
 
   txtorcon = callPackage ../development/python-modules/txtorcon { };
 
+  typecode = callPackage ../development/python-modules/typecode { };
+
+  typecode-libmagic = callPackage ../development/python-modules/typecode/libmagic.nix {
+    inherit (pkgs) file zlib;
+  };
+
   typed-ast = callPackage ../development/python-modules/typed-ast { };
 
   typeguard = callPackage ../development/python-modules/typeguard { };
@@ -8844,6 +8892,8 @@ in {
   else
     callPackage ../development/python-modules/urllib3/2.nix { };
 
+  urlpy = callPackage ../development/python-modules/urlpy { };
+
   urwid = callPackage ../development/python-modules/urwid { };
 
   urwidtrees = callPackage ../development/python-modules/urwidtrees { };