Merge pull request #41080 from dotlambda/home-assistant-recursive-reqs

home-assistant: include requirements of dependencies

pkgs/servers/home-assistant/parse-requirements.py

@@ -1,28 +1,30 @@
 #! /usr/bin/env nix-shell
-#! nix-shell -i python3 -p "python3.withPackages (ps: with ps; [ ])"
+#! nix-shell -i python3 -p "python3.withPackages (ps: with ps; [ requests pyyaml pytz pip jinja2 voluptuous typing aiohttp async-timeout astral certifi attrs ])"
 #
-# This script downloads https://github.com/home-assistant/home-assistant/blob/master/requirements_all.txt.
+# This script downloads Home Assistant's source tarball.
-# This file contains lines of the form
+# Inside the homeassistant/components directory, each component has an associated .py file,
+# specifying required packages and other components it depends on:
 #
-#     # homeassistant.components.foo
+# REQUIREMENTS = [ 'package==1.2.3' ]
-#     # homeassistant.components.bar
+# DEPENDENCIES = [ 'component' ]
-#     foobar==1.2.3
 #
-# i.e. it lists dependencies and the components that require them.
+# By parsing the files, a dictionary mapping component to requirements and dependencies is created.
-# By parsing the file, a dictionary mapping component to dependencies is created.
+# For all of these requirements and the dependencies' requirements,
-# For all of these dependencies, Nixpkgs' python3Packages are searched for appropriate names.
+# Nixpkgs' python3Packages are searched for appropriate names.
 # Then, a Nix attribute set mapping component name to dependencies is created.
 
 from urllib.request import urlopen
-from collections import OrderedDict
+import tempfile
+from io import BytesIO
+import tarfile
+import importlib
 import subprocess
 import os
 import sys
 import json
 import re
 
-GENERAL_PREFIX = '# homeassistant.'
+COMPONENT_PREFIX = 'homeassistant.components'
-COMPONENT_PREFIX = GENERAL_PREFIX + 'components.'
 PKG_SET = 'python3Packages'
 
 # If some requirements are matched by multiple python packages,
@@ -37,28 +39,32 @@ def get_version():
         m = re.search('hassVersion = "([\\d\\.]+)";', f.read())
         return m.group(1)
 
-def fetch_reqs(version='master'):
+def parse_components(version='master'):
-    requirements = {}
+    components = {}
-    with urlopen('https://github.com/home-assistant/home-assistant/raw/{}/requirements_all.txt'.format(version)) as response:
+    with tempfile.TemporaryDirectory() as tmp:
-        components = []
+        with urlopen('https://github.com/home-assistant/home-assistant/archive/{}.tar.gz'.format(version)) as response:
-        for line in response.read().decode().splitlines():
+            tarfile.open(fileobj=BytesIO(response.read())).extractall(tmp)
-            if line == '':
+        # Use part of a script from the Home Assistant codebase
-                components = []
+        sys.path.append(tmp + '/home-assistant-{}'.format(version))
-            elif line[:len(COMPONENT_PREFIX)] == COMPONENT_PREFIX:
+        from script.gen_requirements_all import explore_module
-                component = line[len(COMPONENT_PREFIX):]
+        for package in explore_module(COMPONENT_PREFIX, True):
-                components.append(component)
+            # Remove 'homeassistant.components.' prefix
-                if component not in requirements:
+            component = package[len(COMPONENT_PREFIX + '.'):]
-                    requirements[component] = []
+            try:
-            elif line[:len(GENERAL_PREFIX)] != GENERAL_PREFIX: # skip lines like "# homeassistant.scripts.xyz"
+                module = importlib.import_module(package)
-                # Some dependencies are commented out because they don't build on all platforms
+                components[component] = {}
-                # Since they are still required for running the component, don't skip them
+                components[component]['requirements'] = getattr(module, 'REQUIREMENTS', [])
-                if line[:2] == '# ':
+                components[component]['dependencies'] = getattr(module, 'DEPENDENCIES', [])
-                    line = line[2:]
+            # If there is an ImportError, the imported file is not the main file of the component
-                # Some requirements are specified by url, e.g. https://example.org/foobar#xyz==1.0.0
+            except ImportError:
-                # Therefore, if there's a "#" in the line, only take the part after it
+                continue
-                line = line[line.find('#') + 1:]
+    return components
-                for component in components:
+
-                    requirements[component].append(line)
+# Recursively get the requirements of a component and its dependencies
+def get_reqs(components, component):
+    requirements = set(components[component]['requirements'])
+    for dependency in components[component]['dependencies']:
+        requirements.update(get_reqs(components, dependency))
     return requirements
 
 # Store a JSON dump of Nixpkgs' python3Packages
@@ -95,11 +101,14 @@ def name_to_attr_path(req):
 
 version = get_version()
 print('Generating component-packages.nix for version {}'.format(version))
-requirements = fetch_reqs(version=version)
+components = parse_components(version=version)
 build_inputs = {}
-for component, reqs in OrderedDict(sorted(requirements.items())).items():
+for component in sorted(components.keys()):
     attr_paths = []
-    for req in reqs:
+    for req in sorted(get_reqs(components, component)):
+        # Some requirements are specified by url, e.g. https://example.org/foobar#xyz==1.0.0
+        # Therefore, if there's a "#" in the line, only take the part after it
+        req = req[req.find('#') + 1:]
         name = req.split('==')[0]
         attr_path = name_to_attr_path(name)
         if attr_path is not None:
@@ -108,11 +117,8 @@ for component, reqs in OrderedDict(sorted(requirements.items())).items():
     else:
         build_inputs[component] = attr_paths
 
-# Only select components which have any dependency
-#build_inputs = {k: v for k, v in build_inputs.items() if len(v) > 0}
-
 with open(os.path.dirname(sys.argv[0]) + '/component-packages.nix', 'w') as f:
-    f.write('# Generated from parse-requirements.py\n')
+    f.write('# Generated by parse-requirements.py\n')
     f.write('# Do not edit!\n\n')
     f.write('{\n')
     f.write('  version = "{}";\n'.format(version))