publish.py

#! /usr/bin/env python3

"""This is a script for publishing the wasmer crates to crates.io.
It should be run in the root of wasmer like `python3 scripts/publish.py --help`.

Please lint with pylint and format with black.

$ pylint scripts/publish.py

--------------------------------------------------------------------
Your code has been rated at 10.00/10 (previous run: 10.00/10, +0.00)

$ black scripts/publish.py
"""

import argparse
import itertools
import os
import re
import subprocess
import time
import sys
import typing
from pprint import pprint

try:
    # try import tomllib from standard Python library: New in version 3.11.
    import tomllib
except ModuleNotFoundError:
    try:
        # if tomllib is missing, this is an older python3 version. Try
        # importing the equivalent third-party library tomli:
        import tomli as tomllib
    except ModuleNotFoundError:
        print("Please install tomli, `pip3 install tomli`")
        sys.exit(1)

try:
    from graphlib import TopologicalSorter
except ImportError:
    print("Incompatible python3 version: lacks graphlib standard library module.")
    sys.exit(1)

SETTINGS = {
    # extra features to put when publishing, for example wasmer-cli needs a
    # compiler by default otherwise it won't work standalone
    "publish_features": {
        "wasmer-cli": "default,cranelift",
        "wasmer-wasix": "sys,wasmer/sys",
        "wasmer-wasix-types": "wasmer/sys",
        "wasmer-wasix-experimental-io-devices": "wasmer-wasix/sys,wasmer/sys",
        "wasmer-wast": "wasmer/sys",
        "wai-bindgen-wasmer": "sys",
        "wasmer-cache": "wasmer/sys",
        "wasmer-emscripten": "wasmer/sys"
    },
    # workspace members we want to publish but whose path doesn't start by
    # "./lib/"
    "non-lib-workspace-members": set(["tests/lib/wast"]),
}


def get_latest_version_for_crate(crate_name: str) -> typing.Optional[str]:
    """Fetches the latest version of a given crate name in local cargo registry."""
    output = subprocess.run(
        ["cargo", "search", crate_name], capture_output=True, check=True
    )
    rexp_src = f'^{crate_name} = "([^"]+)"'
    prog = re.compile(rexp_src)
    haystack = output.stdout.decode("utf-8")
    for line in haystack.splitlines():
        result = prog.match(line)
        if result:
            return result.group(1)
    return None


class Crate:
    """Represents a workspace crate that is to be published to crates.io."""

    def __init__(
        self,
        dependencies: typing.List[str],
        cargo_manifest: dict,
        cargo_file_path="Cargo.toml",
    ):
        self.name = cargo_manifest["package"]["name"]
        self.dependencies = dependencies
        self.cargo_manifest = cargo_manifest
        if not os.path.isabs(cargo_file_path):
            cargo_file_path = os.path.join(os.getcwd(), cargo_file_path)
        self.cargo_file_path = cargo_file_path

    def __str__(self):
        return f"{self.name}: {self.dependencies} {self.cargo_file_path} {self.path()}"

    @property
    def path(self) -> str:
        """Return the absolute filesystem path containing this crate."""
        return os.path.dirname(self.cargo_file_path)

    @property
    def version(self) -> str:
        """Return the crate's version according to its manifest."""
        return self.cargo_manifest["package"]["version"]


class Publisher:
    """A class responsible for figuring out dependencies,
    creating a topological sorting in order to publish them
    to crates.io in a valid order."""

    def __init__(self, version=None, dry_run=True, verbose=True):
        self.dry_run: bool = dry_run
        self.verbose: bool = verbose

        # open workspace Cargo.toml
        with open("Cargo.toml", "rb") as file:
            data = tomllib.load(file)

        if version is None:
            version = data["package"]["version"]
        self.version: str = version

        if self.verbose and not self.dry_run:
            print(f"Publishing version {self.version}")
        elif self.verbose and self.dry_run:
            print(f"Publishing version {self.version} dry run!")

        # define helper function
        check_local_dep_fn = lambda t: isinstance(t[1], dict) and "path" in t[1]
        members = set(
            map(
                lambda p: p + "/Cargo.toml",
                filter(
                    lambda path: (
                        path.startswith("lib/") and os.path.exists(path + "/Cargo.toml")
                    )
                    or path in SETTINGS["non-lib-workspace-members"],
                    itertools.chain(
                        data["workspace"]["members"],
                        map(
                            lambda p: p[1]["path"],
                            filter(check_local_dep_fn, data["dependencies"].items()),
                        ),
                    ),
                ),
            )
        )
        crates = []
        for member in members:
            with open(member, "rb") as file:
                member_data = tomllib.load(file)

            def return_dependencies(toml) -> typing.List[str]:
                acc = set()
                stack = [toml]
                while len(stack) > 0:
                    toml = stack.pop()
                    if "dependencies" in toml:
                        acc.update(
                            list(
                                map(
                                    lambda dep: dep[1]["package"]
                                    if "package" in dep[1]
                                    else dep[0],
                                    filter(
                                        check_local_dep_fn, toml["dependencies"].items()
                                    ),
                                )
                            )
                        )
                    if "target" in toml:
                        stack.append(toml["target"])
                    for key, value in toml.items():
                        if key.startswith("cfg"):
                            stack.append(value)
                return list(acc)

            dependencies = return_dependencies(member_data)
            crates.append(Crate(dependencies, member_data, cargo_file_path=member))

        self.crates = crates
        self.crate_index: typing.Dict[str, Crate] = {c.name: c for c in crates}

        self.create_publish_order()

        self.starting_dir = os.getcwd()

    def create_publish_order(self):
        """Creates a valid publish order by topologically
        sorting crates using the dependency graph."""
        topological_sorter = TopologicalSorter()

        for crate in self.crates:
            topological_sorter.add(crate.name, *crate.dependencies)

        self.publish_order: typing.List[str] = [*topological_sorter.static_order()]

    def is_crate_already_published(self, crate_name: str) -> bool:
        """Checks if a given crate name is already published
        with the version string we intend to publish with."""
        found_string: str = get_latest_version_for_crate(crate_name)
        if found_string is None:
            return False

        return self.version == found_string

    def publish_crate(self, crate_name: str):
        # pylint: disable=broad-except
        """Publish a given crate by name."""
        status = None
        try:
            crate = self.crate_index[crate_name]
            os.chdir(crate.path)
            extra_args = []
            if crate_name in SETTINGS["publish_features"]:
                extra_args = ["--features", SETTINGS["publish_features"][crate_name]]
            if self.dry_run:
                print(f"In dry-run: not publishing crate `{crate_name}`")
                command = ["cargo", "publish", "--dry-run"] + extra_args
                if self.verbose:
                    print(*command)
                output = subprocess.run(command, check=True)
                if self.verbose:
                    print(output)
            else:
                command = ["cargo", "publish"] + extra_args
                if self.verbose:
                    print(*command)
                output = subprocess.run(command, check=True)
                if self.verbose:
                    print(output)
            if self.verbose:
                print("Success.")
        except Exception as exc:
            if self.verbose:
                print(f"Failed to publish {crate_name}.")
            print(exc)
            status = exc
        finally:
            os.chdir(self.starting_dir)
        return status

    def publish(self):
        # pylint: disable=too-many-branches
        """Publish all packages in workspace."""
        if self.verbose and self.dry_run:
            print("(Dry run, not actually publishing anything)")
        if self.verbose:
            print("Publishing order:")
            pprint(self.publish_order)
        status = {}
        failures = 0

        for crate_name in self.publish_order:
            print(f"Publishing `{crate_name}`...")
            if not self.is_crate_already_published(crate_name):
                status[crate_name] = self.publish_crate(crate_name)
                if status[crate_name]:
                    failures = +1
            else:
                print(f"`{crate_name}` was already published!")
                continue

            # sleep for 16 seconds between crates to ensure the crates.io
            # index has time to update
            if not self.dry_run:
                print(
                    "Sleeping for 16 seconds to allow the `crates.io` index to update..."
                )
                time.sleep(16)
            else:
                print("In dry-run: not sleeping for crates.io to update.")
        if failures > 0 and self.verbose:
            print(f"encountered {failures} failures.")
            for key, value in status.items():
                if value is None:
                    result = "ok"
                else:
                    result = str(value)
                print(f"{key}\t{result}")
        if self.verbose:
            print(f"Published {len(status) - failures} crates.")
        return failures


def main():
    """Main executable function."""
    os.environ["WASMER_PUBLISH_SCRIPT_IS_RUNNING"] = "1"
    parser = argparse.ArgumentParser(
        description="Publish the Wasmer crates to crates.io"
    )
    subparsers = parser.add_subparsers(dest="subcommand")
    health_cmd = subparsers.add_parser(
        "health-check",
        help="""Check the dependency graph is a tree, meaning a non-cyclic
        planar graph. Combine with verbosity to print a topological sorting
        of the graph.""",
    )
    health_cmd.add_argument(
        "-v",
        "--verbose",
        default=False,
        action="store_true",
        help="Be verbose.",
    )
    health_cmd.add_argument(
        "--print-dependencies",
        default=False,
        action="store_true",
        help="For each crate, print its dependencies.",
    )
    publish_cmd = subparsers.add_parser(
        "publish", help="Publish Wasmer crates to crates.io."
    )
    publish_cmd.add_argument(
        "--version",
        default=None,
        type=str,
        help="""Define the semver target triple (Default is automatically
        read from workspace Cargo.toml.""",
    )
    publish_cmd.add_argument(
        "--dry-run",
        default=False,
        action="store_true",
        help="""Run the script without actually publishing anything to
        crates.io""",
    )
    publish_cmd.add_argument(
        "-v",
        "--verbose",
        default=False,
        action="store_true",
        help="Be verbose.",
    )

    args = parser.parse_args()
    if args.subcommand == "health-check":
        verbose = args.verbose
        publisher = Publisher(verbose=verbose)
        if verbose:
            print(f"Version is {publisher.version}")
            pprint(publisher.publish_order)
        if args.print_dependencies:
            for crate in publisher.crates:
                print(f"{crate.name}: {crate.dependencies}")
        return 0

    if args.subcommand == "publish":
        verbose = args.verbose
        publisher = Publisher(dry_run=args.dry_run, verbose=verbose)
        return publisher.publish()
    return 0


if __name__ == "__main__":
    main()