nixpkgs/pkgs/servers/home-assistant/parse-requirements.py

#! /usr/bin/env nix-shell
#! nix-shell -i python3 -p "python3.withPackages (ps: with ps; [ ])"
#
# This script downloads https://github.com/home-assistant/home-assistant/blob/master/requirements_all.txt.
# This file contains lines of the form
#
#     # homeassistant.components.foo
#     # homeassistant.components.bar
#     foobar==1.2.3
#
# i.e. it lists dependencies and the components that require them.
# By parsing the file, a dictionary mapping component to dependencies is created.
# For all of these dependencies, Nixpkgs' python3Packages are searched for appropriate names.
# Then, a Nix attribute set mapping component name to dependencies is created.

from urllib.request import urlopen
from collections import OrderedDict
import subprocess
import os
import sys
import json
import re

GENERAL_PREFIX = '# homeassistant.'
COMPONENT_PREFIX = GENERAL_PREFIX + 'components.'
PKG_SET = 'python3Packages'

# If some requirements are matched by multiple python packages,
# the following can be used to choose one of them
PKG_PREFERENCES = {
    # Use python3Packages.youtube-dl-light instead of python3Packages.youtube-dl
    'youtube-dl': 'youtube-dl-light'
}

def get_version():
    with open(os.path.dirname(sys.argv[0]) + '/default.nix') as f:
        m = re.search('hassVersion = "([\\d\\.]+)";', f.read())
        return m.group(1)

def fetch_reqs(version='master'):
    requirements = {}
    with urlopen('https://github.com/home-assistant/home-assistant/raw/{}/requirements_all.txt'.format(version)) as response:
        components = []
        for line in response.read().decode().splitlines():
            if line == '':
                components = []
            elif line[:len(COMPONENT_PREFIX)] == COMPONENT_PREFIX:
                component = line[len(COMPONENT_PREFIX):]
                components.append(component)
                if component not in requirements:
                    requirements[component] = []
            elif line[:len(GENERAL_PREFIX)] != GENERAL_PREFIX: # skip lines like "# homeassistant.scripts.xyz"
                # Some dependencies are commented out because they don't build on all platforms
                # Since they are still required for running the component, don't skip them
                if line[:2] == '# ':
                    line = line[2:]
                # Some requirements are specified by url, e.g. https://example.org/foobar#xyz==1.0.0
                # Therefore, if there's a "#" in the line, only take the part after it
                line = line[line.find('#') + 1:]
                for component in components:
                    requirements[component].append(line)
    return requirements

# Store a JSON dump of Nixpkgs' python3Packages
output = subprocess.check_output(['nix-env', '-f', os.path.dirname(sys.argv[0]) + '/../../..', '-qa', '-A', PKG_SET, '--json'])
packages = json.loads(output)

def name_to_attr_path(req):
    attr_paths = set()
    names = [req]
    # E.g. python-mpd2 is actually called python3.6-mpd2
    # instead of python-3.6-python-mpd2 inside Nixpkgs
    if req.startswith('python-') or req.startswith('python_'):
        names.append(req[len('python-'):])
    for name in names:
        # treat "-" and "_" equally
        name = re.sub('[-_]', '[-_]', name)
        pattern = re.compile('^python\\d\\.\\d-{}-\\d'.format(name), re.I)
        for attr_path, package in packages.items():
            if pattern.match(package['name']):
                attr_paths.add(attr_path)
    if len(attr_paths) > 1:
        for to_replace, replacement in PKG_PREFERENCES.items():
            try:
                attr_paths.remove(PKG_SET + '.' + to_replace)
                attr_paths.add(PKG_SET + '.' + replacement)
            except KeyError:
                pass
    # Let's hope there's only one derivation with a matching name
    assert(len(attr_paths) <= 1)
    if len(attr_paths) == 1:
        return attr_paths.pop()
    else:
        return None

version = get_version()
print('Generating component-packages.nix for version {}'.format(version))
requirements = fetch_reqs(version=version)
build_inputs = {}
for component, reqs in OrderedDict(sorted(requirements.items())).items():
    attr_paths = []
    for req in reqs:
        name = req.split('==')[0]
        attr_path = name_to_attr_path(name)
        if attr_path is not None:
            # Add attribute path without "python3Packages." prefix
            attr_paths.append(attr_path[len(PKG_SET + '.'):])
    else:
        build_inputs[component] = attr_paths

# Only select components which have any dependency
#build_inputs = {k: v for k, v in build_inputs.items() if len(v) > 0}

with open(os.path.dirname(sys.argv[0]) + '/component-packages.nix', 'w') as f:
    f.write('# Generated from parse-requirements.py\n')
    f.write('# Do not edit!\n\n')
    f.write('{\n')
    f.write('  version = "{}";\n'.format(version))
    f.write('  components = {\n')
    for component, attr_paths in build_inputs.items():
        f.write('    "{}" = ps: with ps; [ '.format(component))
        f.write(' '.join(attr_paths))
        f.write(' ];\n')
    f.write('  };\n')
    f.write('}\n')