JakeHillion/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #86074 from emilazy/refactor-linux-hardened-update-script

Browse Source
This commit is contained in:
Jörg Thalheim 2020-05-11 10:40:42 +01:00 committed by GitHub
commit 885f65fbff
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 284 additions and 236 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/kernel.nix
View File

@ -14,7 +14,7 @@ with lib;
freeform = x: { freeform = x; };
/*
Common patterns/legacy used in common-config/hardened-config.nix
Common patterns/legacy used in common-config/hardened/config.nix
*/
whenHelpers = version: {
whenAtLeast = ver: mkIf (versionAtLeast version ver);

4
pkgs/development/python-modules/pyGithub/default.nix
View File

@ -12,14 +12,14 @@
buildPythonPackage rec {
pname = "PyGithub";
version = "1.47";
version = "1.51";
disabled = !isPy3k;
src = fetchFromGitHub {
owner = "PyGithub";
repo = "PyGithub";
rev = "v${version}";
sha256 = "0zvp1gib2lryw698vxkbdv40n3lsmdlhwp7vdcg41dqqa5nfryhn";
hash = "sha256-8uQCFiw1ByPOX8ZRUlSLYPIibjmd19r/JtTnmQdz5cM=";
};
checkInputs = [ httpretty parameterized pytestCheckHook ];

0
pkgs/os-specific/linux/kernel/anthraxx.asc → pkgs/os-specific/linux/kernel/hardened/anthraxx.asc
View File

0
pkgs/os-specific/linux/kernel/hardened-config.nix → pkgs/os-specific/linux/kernel/hardened/config.nix
View File

0
pkgs/os-specific/linux/kernel/hardened-patches.json → pkgs/os-specific/linux/kernel/hardened/patches.json
View File

0
pkgs/os-specific/linux/kernel/tag-hardened.patch → pkgs/os-specific/linux/kernel/hardened/tag-hardened.patch
View File

277
pkgs/os-specific/linux/kernel/hardened/update.py Executable file
View File

@ -0,0 +1,277 @@
#! /usr/bin/env nix-shell
#! nix-shell -i python -p "python38.withPackages (ps: [ps.PyGithub])" git gnupg
# This is automatically called by ../update.sh.
from __future__ import annotations
import json
import os
import re
import subprocess
import sys
from dataclasses import dataclass
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import (
Dict,
Iterator,
List,
Optional,
Sequence,
Tuple,
TypedDict,
Union,
)
from github import Github
from github.GitRelease import GitRelease
VersionComponent = Union[int, str]
Version = List[VersionComponent]
Patch = TypedDict("Patch", {"name": str, "url": str, "sha256": str})
@dataclass
class ReleaseInfo:
version: Version
release: GitRelease
HERE = Path(__file__).resolve().parent
NIXPKGS_KERNEL_PATH = HERE.parent
NIXPKGS_PATH = HERE.parents[4]
HARDENED_GITHUB_REPO = "anthraxx/linux-hardened"
HARDENED_TRUSTED_KEY = HERE / "anthraxx.asc"
HARDENED_PATCHES_PATH = HERE / "patches.json"
MIN_KERNEL_VERSION: Version = [4, 14]
def run(*args: Union[str, Path]) -> subprocess.CompletedProcess[bytes]:
try:
return subprocess.run(
args,
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
encoding="utf-8",
)
except subprocess.CalledProcessError as err:
print(
f"error: `{err.cmd}` failed unexpectedly\n"
f"status code: {err.returncode}\n"
f"stdout:\n{err.stdout.strip()}\n"
f"stderr:\n{err.stderr.strip()}",
file=sys.stderr,
)
sys.exit(1)
def nix_prefetch_url(url: str) -> Tuple[str, Path]:
output = run("nix-prefetch-url", "--print-path", url).stdout
sha256, path = output.strip().split("\n")
return sha256, Path(path)
def verify_openpgp_signature(
*, name: str, trusted_key: Path, sig_path: Path, data_path: Path,
) -> bool:
with TemporaryDirectory(suffix=".nixpkgs-gnupg-home") as gnupg_home_str:
gnupg_home = Path(gnupg_home_str)
run("gpg", "--homedir", gnupg_home, "--import", trusted_key)
keyring = gnupg_home / "pubring.kbx"
try:
subprocess.run(
("gpgv", "--keyring", keyring, sig_path, data_path),
check=True,
stderr=subprocess.PIPE,
encoding="utf-8",
)
return True
except subprocess.CalledProcessError as err:
print(
f"error: signature for {name} failed to verify!",
file=sys.stderr,
)
print(err.stderr, file=sys.stderr, end="")
return False
def fetch_patch(*, name: str, release: GitRelease) -> Optional[Patch]:
def find_asset(filename: str) -> str:
try:
it: Iterator[str] = (
asset.browser_download_url
for asset in release.get_assets()
if asset.name == filename
)
return next(it)
except StopIteration:
raise KeyError(filename)
patch_filename = f"{name}.patch"
try:
patch_url = find_asset(patch_filename)
sig_url = find_asset(patch_filename + ".sig")
except KeyError:
print(f"error: {patch_filename}{{,.sig}} not present", file=sys.stderr)
return None
sha256, patch_path = nix_prefetch_url(patch_url)
_, sig_path = nix_prefetch_url(sig_url)
sig_ok = verify_openpgp_signature(
name=name,
trusted_key=HARDENED_TRUSTED_KEY,
sig_path=sig_path,
data_path=patch_path,
)
if not sig_ok:
return None
return Patch(name=patch_filename, url=patch_url, sha256=sha256)
def parse_version(version_str: str) -> Version:
version: Version = []
for component in version_str.split("."):
try:
version.append(int(component))
except ValueError:
version.append(component)
return version
def version_string(version: Version) -> str:
return ".".join(str(component) for component in version)
def major_kernel_version_key(kernel_version: Version) -> str:
return version_string(kernel_version[:-1])
def commit_patches(*, kernel_key: str, message: str) -> None:
new_patches_path = HARDENED_PATCHES_PATH.with_suffix(".new")
with open(new_patches_path, "w") as new_patches_file:
json.dump(patches, new_patches_file, indent=4, sort_keys=True)
new_patches_file.write("\n")
os.rename(new_patches_path, HARDENED_PATCHES_PATH)
message = f"linux/hardened/patches/{kernel_key}: {message}"
print(message)
if os.environ.get("COMMIT"):
run(
"git",
"-C",
NIXPKGS_PATH,
"commit",
f"--message={message}",
HARDENED_PATCHES_PATH,
)
# Load the existing patches.
patches: Dict[str, Patch]
with open(HARDENED_PATCHES_PATH) as patches_file:
patches = json.load(patches_file)
# Get the set of currently packaged kernel versions.
kernel_versions = {}
for filename in os.listdir(NIXPKGS_KERNEL_PATH):
filename_match = re.fullmatch(r"linux-(\d+)\.(\d+)\.nix", filename)
if filename_match:
nix_version_expr = f"""
with import {NIXPKGS_PATH} {{}};
(callPackage {NIXPKGS_KERNEL_PATH / filename} {{}}).version
"""
kernel_version = parse_version(
run(
"nix", "eval", "--impure", "--raw", "--expr", nix_version_expr,
).stdout
)
if kernel_version < MIN_KERNEL_VERSION:
continue
kernel_key = major_kernel_version_key(kernel_version)
kernel_versions[kernel_key] = kernel_version
# Remove patches for unpackaged kernel versions.
for kernel_key in sorted(patches.keys() - kernel_versions.keys()):
commit_patches(kernel_key=kernel_key, message="remove")
g = Github(os.environ.get("GITHUB_TOKEN"))
repo = g.get_repo(HARDENED_GITHUB_REPO)
failures = False
# Match each kernel version with the best patch version.
releases = {}
for release in repo.get_releases():
version = parse_version(release.tag_name)
# needs to look like e.g. 5.6.3.a
if len(version) < 4:
continue
kernel_version = version[:-1]
kernel_key = major_kernel_version_key(kernel_version)
try:
packaged_kernel_version = kernel_versions[kernel_key]
except KeyError:
continue
release_info = ReleaseInfo(version=version, release=release)
if kernel_version == packaged_kernel_version:
releases[kernel_key] = release_info
else:
# Fall back to the latest patch for this major kernel version,
# skipping patches for kernels newer than the packaged one.
if kernel_version > packaged_kernel_version:
continue
elif (
kernel_key not in releases or releases[kernel_key].version < version
):
releases[kernel_key] = release_info
# Update hardened-patches.json for each release.
for kernel_key in sorted(releases.keys()):
release_info = releases[kernel_key]
release = release_info.release
version = release_info.version
version_str = release.tag_name
name = f"linux-hardened-{version_str}"
old_version: Optional[Version] = None
old_version_str: Optional[str] = None
update: bool
try:
old_filename = patches[kernel_key]["name"]
old_version_str = old_filename.replace("linux-hardened-", "").replace(
".patch", ""
)
old_version = parse_version(old_version_str)
update = old_version < version
except KeyError:
update = True
if update:
patch = fetch_patch(name=name, release=release)
if patch is None:
failures = True
else:
patches[kernel_key] = patch
if old_version:
message = f"{old_version_str} -> {version_str}"
else:
message = f"init at {version_str}"
commit_patches(kernel_key=kernel_key, message=message)
missing_kernel_versions = kernel_versions.keys() - patches.keys()
if missing_kernel_versions:
print(
f"warning: no patches for kernel versions "
+ ", ".join(missing_kernel_versions),
file=sys.stderr,
)
if failures:
sys.exit(1)

4
pkgs/os-specific/linux/kernel/patches.nix
View File

@ -35,7 +35,7 @@
tag_hardened = {
name = "tag-hardened";
patch = ./tag-hardened.patch;
patch = ./hardened/tag-hardened.patch;
};
hardened = let
@ -43,7 +43,7 @@
name = lib.removeSuffix ".patch" src.name;
patch = fetchurl src;
};
patches = builtins.fromJSON (builtins.readFile ./hardened-patches.json);
patches = builtins.fromJSON (builtins.readFile ./hardened/patches.json);
in lib.mapAttrs mkPatch patches;
# https://bugzilla.kernel.org/show_bug.cgi?id=197591#c6

229
pkgs/os-specific/linux/kernel/update-hardened.py
View File

@ -1,229 +0,0 @@
#! /usr/bin/env nix-shell
#! nix-shell -i python -p "python3.withPackages (ps: [ps.PyGithub])" git gnupg
# This is automatically called by ./update.sh.
import re
import json
import sys
import os.path
from glob import glob
import subprocess
from tempfile import TemporaryDirectory
from github import Github
HERE = os.path.dirname(os.path.realpath(__file__))
HARDENED_GITHUB_REPO = 'anthraxx/linux-hardened'
HARDENED_TRUSTED_KEY = os.path.join(HERE, 'anthraxx.asc')
HARDENED_PATCHES_PATH = os.path.join(HERE, 'hardened-patches.json')
MIN_KERNEL_VERSION = [4, 14]
def run(*args, **kwargs):
try:
return subprocess.run(
args, **kwargs,
check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
)
except subprocess.CalledProcessError as err:
print(
f'error: `{err.cmd}` failed unexpectedly\n'
f'status code: {err.returncode}\n'
f'stdout:\n{err.stdout.decode("utf-8").strip()}\n'
f'stderr:\n{err.stderr.decode("utf-8").strip()}',
file=sys.stderr,
)
sys.exit(1)
def nix_prefetch_url(url):
output = run('nix-prefetch-url', '--print-path', url).stdout
return output.decode('utf-8').strip().split('\n')
def verify_openpgp_signature(*, name, trusted_key, sig_path, data_path):
with TemporaryDirectory(suffix='.nixpkgs-gnupg-home') as gnupg_home:
run('gpg', '--homedir', gnupg_home, '--import', trusted_key)
keyring = os.path.join(gnupg_home, 'pubring.kbx')
try:
subprocess.run(
('gpgv', '--keyring', keyring, sig_path, data_path),
check=True, stderr=subprocess.PIPE,
)
return True
except subprocess.CalledProcessError as err:
print(
f'error: signature for {name} failed to verify!',
file=sys.stderr,
)
print(err.stderr.decode('utf-8'), file=sys.stderr, end='')
return False
def fetch_patch(*, name, release):
def find_asset(filename):
try:
return next(
asset.browser_download_url
for asset in release.get_assets()
if asset.name == filename
)
except StopIteration:
raise KeyError(filename)
patch_filename = f'{name}.patch'
try:
patch_url = find_asset(patch_filename)
sig_url = find_asset(patch_filename + '.sig')
except KeyError:
print(f'error: {patch_filename}{{,.sig}} not present', file=sys.stderr)
return None
sha256, patch_path = nix_prefetch_url(patch_url)
_, sig_path = nix_prefetch_url(sig_url)
sig_ok = verify_openpgp_signature(
name=name,
trusted_key=HARDENED_TRUSTED_KEY,
sig_path=sig_path,
data_path=patch_path,
)
if not sig_ok:
return None
return {
'name': patch_filename,
'url': patch_url,
'sha256': sha256,
}
def parse_version(version_str):
version = []
for component in version_str.split('.'):
try:
version.append(int(component))
except ValueError:
version.append(component)
return version
def version_string(version):
return '.'.join(str(component) for component in version)
def major_kernel_version_key(kernel_version):
return version_string(kernel_version[:-1])
def commit_patches(*, kernel_key, message):
with open(HARDENED_PATCHES_PATH + '.new', 'w') as new_patches_file:
json.dump(patches, new_patches_file, indent=4, sort_keys=True)
new_patches_file.write('\n')
os.rename(HARDENED_PATCHES_PATH + '.new', HARDENED_PATCHES_PATH)
message = f'linux/hardened-patches/{kernel_key}: {message}'
print(message)
if os.environ.get('COMMIT'):
run(
'git', '-C', HERE, 'commit', f'--message={message}',
'hardened-patches.json',
)
# Load the existing patches.
with open(HARDENED_PATCHES_PATH) as patches_file:
patches = json.load(patches_file)
NIX_VERSION_RE = re.compile(r'''
\s* version \s* =
\s* " (?P<version> [^"]*) "
\s* ; \s* \n
''', re.VERBOSE)
# Get the set of currently packaged kernel versions.
kernel_versions = {}
for filename in os.listdir(HERE):
filename_match = re.fullmatch(r'linux-(\d+)\.(\d+)\.nix', filename)
if filename_match:
with open(os.path.join(HERE, filename)) as nix_file:
for nix_line in nix_file:
match = NIX_VERSION_RE.fullmatch(nix_line)
if match:
kernel_version = parse_version(match.group('version'))
if kernel_version < MIN_KERNEL_VERSION:
continue
kernel_key = major_kernel_version_key(kernel_version)
kernel_versions[kernel_key] = kernel_version
# Remove patches for unpackaged kernel versions.
for kernel_key in sorted(patches.keys() - kernel_versions.keys()):
commit_patches(kernel_key=kernel_key, message='remove')
g = Github(os.environ.get('GITHUB_TOKEN'))
repo = g.get_repo(HARDENED_GITHUB_REPO)
failures = False
# Match each kernel version with the best patch version.
releases = {}
for release in repo.get_releases():
version = parse_version(release.tag_name)
# needs to look like e.g. 5.6.3.a
if len(version) < 4:
continue
kernel_version = version[:-1]
kernel_key = major_kernel_version_key(kernel_version)
try:
packaged_kernel_version = kernel_versions[kernel_key]
except KeyError:
continue
release_info = {
'version': version,
'release': release,
}
if kernel_version == packaged_kernel_version:
releases[kernel_key] = release_info
else:
# Fall back to the latest patch for this major kernel version,
# skipping patches for kernels newer than the packaged one.
if kernel_version > packaged_kernel_version:
continue
elif (kernel_key not in releases or
releases[kernel_key]['version'] < version):
releases[kernel_key] = release_info
# Update hardened-patches.json for each release.
for kernel_key, release_info in releases.items():
release = release_info['release']
version = release_info['version']
version_str = release.tag_name
name = f'linux-hardened-{version_str}'
try:
old_filename = patches[kernel_key]['name']
old_version_str = (old_filename
.replace('linux-hardened-', '')
.replace('.patch', ''))
old_version = parse_version(old_version_str)
update = old_version < version
except KeyError:
update = True
old_version = None
if update:
patch = fetch_patch(name=name, release=release)
if patch is None:
failures = True
else:
patches[kernel_key] = patch
if old_version:
message = f'{old_version_str} -> {version_str}'
else:
message = f'init at {version_str}'
commit_patches(kernel_key=kernel_key, message=message)
missing_kernel_versions = kernel_versions.keys() - patches.keys()
if missing_kernel_versions:
print(
f'warning: no patches for kernel versions ' +
', '.join(missing_kernel_versions),
file=sys.stderr,
)
if failures:
sys.exit(1)

2
pkgs/os-specific/linux/kernel/update.sh
View File

@ -62,4 +62,4 @@ done
COMMIT=1 $NIXPKGS/pkgs/os-specific/linux/kernel/update-libre.sh
# Update linux-hardened
COMMIT=1 $NIXPKGS/pkgs/os-specific/linux/kernel/update-hardened.py
COMMIT=1 $NIXPKGS/pkgs/os-specific/linux/kernel/hardened/update.py

2
pkgs/top-level/all-packages.nix
View File

@ -17065,7 +17065,7 @@ in
# Hardened linux
hardenedLinuxPackagesFor = kernel: linuxPackagesFor (kernel.override {
structuredExtraConfig = import ../os-specific/linux/kernel/hardened-config.nix {
structuredExtraConfig = import ../os-specific/linux/kernel/hardened/config.nix {
inherit stdenv;
inherit (kernel) version;
};