Add option to extract licenses [ci] (home-assistant#129095)

.github/workflows/ci.yaml

@@ -615,6 +615,10 @@ jobs:
         && github.event.inputs.mypy-only != 'true'
         || github.event.inputs.audit-licenses-only == 'true')
       && needs.info.outputs.requirements == 'true'
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ${{ fromJson(needs.info.outputs.python_versions) }}
     steps:
       - name: Check out code from GitHub
         uses: actions/[email protected]
@@ -633,19 +637,19 @@ jobs:
           key: >-
             ${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{
             needs.info.outputs.python_cache_key }}
-      - name: Run pip-licenses
+      - name: Extract license data
         run: |
           . venv/bin/activate
-          pip-licenses --format=json --output-file=licenses.json
+          python -m script.licenses extract --output-file=licenses-${{ matrix.python-version }}.json
       - name: Upload licenses
         uses: actions/[email protected]
         with:
-          name: licenses
-          path: licenses.json
-      - name: Process licenses
+          name: licenses-${{ github.run_number }}-${{ matrix.python-version }}
+          path: licenses-${{ matrix.python-version }}.json
+      - name: Check licenses
         run: |
           . venv/bin/activate
-          python -m script.licenses licenses.json
+          python -m script.licenses check licenses-${{ matrix.python-version }}.json
 
   pylint:
     name: Check pylint

requirements_test.txt

@@ -17,7 +17,6 @@ pydantic==1.10.18
 pylint==3.3.1
 pylint-per-file-ignores==1.3.2
 pipdeptree==2.23.4
-pip-licenses==5.0.0
 pytest-asyncio==0.24.0
 pytest-aiohttp==1.0.5
 pytest-cov==5.0.0

script/licenses.py

@@ -2,16 +2,28 @@
 
 from __future__ import annotations
 
-from argparse import ArgumentParser
+from argparse import ArgumentParser, Namespace
 from collections.abc import Sequence
 from dataclasses import dataclass
+from importlib import metadata
 import json
 from pathlib import Path
 import sys
+from typing import TypedDict, cast
 
 from awesomeversion import AwesomeVersion
 
 
+class PackageMetadata(TypedDict):
+    """Package metadata."""
+
+    name: str
+    version: str
+    license_expression: str | None
+    license_metadata: str | None
+    license_classifier: list[str]
+
+
 @dataclass
 class PackageDefinition:
     """Package definition."""
@@ -21,12 +33,16 @@ class PackageDefinition:
     version: AwesomeVersion
 
     @classmethod
-    def from_dict(cls, data: dict[str, str]) -> PackageDefinition:
-        """Create a package definition from a dictionary."""
+    def from_dict(cls, data: PackageMetadata) -> PackageDefinition:
+        """Create a package definition from PackageMetadata."""
+        if not (license_str := "; ".join(data["license_classifier"])):
+            license_str = (
+                data["license_metadata"] or data["license_expression"] or "UNKNOWN"
+            )
         return cls(
-            license=data["License"],
-            name=data["Name"],
-            version=AwesomeVersion(data["Version"]),
+            license=license_str,
+            name=data["name"],
+            version=AwesomeVersion(data["version"]),
         )
 
 
@@ -128,7 +144,6 @@ def from_dict(cls, data: dict[str, str]) -> PackageDefinition:
     "aioecowitt",  # https://github.com/home-assistant-libs/aioecowitt/pull/180
     "aioopenexchangerates",  # https://github.com/MartinHjelmare/aioopenexchangerates/pull/94
     "aiooui",  # https://github.com/Bluetooth-Devices/aiooui/pull/8
-    "aioruuvigateway",  # https://github.com/akx/aioruuvigateway/pull/6
     "apple_weatherkit",  # https://github.com/tjhorner/python-weatherkit/pull/3
     "asyncio",  # PSF License
     "chacha20poly1305",  # LGPL
@@ -159,14 +174,10 @@ def from_dict(cls, data: dict[str, str]) -> PackageDefinition:
     "pyvera",  # https://github.com/maximvelichko/pyvera/pull/164
     "pyxeoma",  # https://github.com/jeradM/pyxeoma/pull/11
     "repoze.lru",
-    "ruuvitag-ble",  # https://github.com/Bluetooth-Devices/ruuvitag-ble/pull/10
-    "sensirion-ble",  # https://github.com/akx/sensirion-ble/pull/9
     "sharp_aquos_rc",  # https://github.com/jmoore987/sharp_aquos_rc/pull/14
     "tapsaff",  # https://github.com/bazwilliams/python-taps-aff/pull/5
     "vincenty",  # Public domain
     "zeversolar",  # https://github.com/kvanzuijlen/zeversolar/pull/46
-    # Using License-Expression (with hatchling)
-    "ftfy",  # Apache-2.0
 }
 
 TODO = {
@@ -176,22 +187,9 @@ def from_dict(cls, data: dict[str, str]) -> PackageDefinition:
 }
 
 
-def main(argv: Sequence[str] | None = None) -> int:
-    """Run the main script."""
+def check_licenses(args: CheckArgs) -> int:
+    """Check licenses are OSI approved."""
     exit_code = 0
-
-    parser = ArgumentParser()
-    parser.add_argument(
-        "path",
-        nargs="?",
-        metavar="PATH",
-        default="licenses.json",
-        help="Path to json licenses file",
-    )
-
-    argv = argv or sys.argv[1:]
-    args = parser.parse_args(argv)
-
     raw_licenses = json.loads(Path(args.path).read_text())
     package_definitions = [PackageDefinition.from_dict(data) for data in raw_licenses]
     for package in package_definitions:
@@ -244,8 +242,92 @@ def main(argv: Sequence[str] | None = None) -> int:
     return exit_code
 
 
+def extract_licenses(args: ExtractArgs) -> int:
+    """Extract license data for installed packages."""
+    licenses = sorted(
+        [get_package_metadata(dist) for dist in list(metadata.distributions())],
+        key=lambda dist: dist["name"],
+    )
+    Path(args.output_file).write_text(json.dumps(licenses, indent=2))
+    return 0
+
+
+def get_package_metadata(dist: metadata.Distribution) -> PackageMetadata:
+    """Get package metadata for distribution."""
+    return {
+        "name": dist.name,
+        "version": dist.version,
+        "license_expression": dist.metadata.get("License-Expression"),
+        "license_metadata": dist.metadata.get("License"),
+        "license_classifier": extract_license_classifier(
+            dist.metadata.get_all("Classifier")
+        ),
+    }
+
+
+def extract_license_classifier(classifiers: list[str] | None) -> list[str]:
+    """Extract license from list of classifiers.
+
+    E.g. 'License :: OSI Approved :: MIT License' -> 'MIT License'.
+    Filter out bare 'License :: OSI Approved'.
+    """
+    return [
+        license_classifier
+        for classifier in classifiers or ()
+        if classifier.startswith("License")
+        and (license_classifier := classifier.rpartition(" :: ")[2])
+        and license_classifier != "OSI Approved"
+    ]
+
+
+class ExtractArgs(Namespace):
+    """Extract arguments."""
+
+    output_file: str
+
+
+class CheckArgs(Namespace):
+    """Check arguments."""
+
+    path: str
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    """Run the main script."""
+    parser = ArgumentParser()
+    subparsers = parser.add_subparsers(title="Subcommands", required=True)
+
+    parser_extract = subparsers.add_parser("extract")
+    parser_extract.set_defaults(action="extract")
+    parser_extract.add_argument(
+        "--output-file",
+        default="licenses.json",
+        help="Path to store the licenses file",
+    )
+
+    parser_check = subparsers.add_parser("check")
+    parser_check.set_defaults(action="check")
+    parser_check.add_argument(
+        "path",
+        nargs="?",
+        metavar="PATH",
+        default="licenses.json",
+        help="Path to json licenses file",
+    )
+
+    argv = argv or sys.argv[1:]
+    args = parser.parse_args(argv)
+
+    if args.action == "extract":
+        args = cast(ExtractArgs, args)
+        return extract_licenses(args)
+    if args.action == "check":
+        args = cast(CheckArgs, args)
+        if (exit_code := check_licenses(args)) == 0:
+            print("All licenses are approved!")
+        return exit_code
+    return 0
+
+
 if __name__ == "__main__":
-    exit_code = main()
-    if exit_code == 0:
-        print("All licenses are approved!")
-    sys.exit(exit_code)
+    sys.exit(main())