Skip to content

Commit

Permalink
treewide: migrate external rules to bzlmod
Browse files Browse the repository at this point in the history
This is a huge one as it was very annoying to migrate them separately. This migrates rules_go, gazelle, rust_rust, protobuf to bzlmod

Change-Id: If39591d43ed4c2afa2979ee5915e9d1cfa1574a9
Reviewed-on: https://review.monogon.dev/c/monogon/+/3234
Tested-by: Jenkins CI
Reviewed-by: Leopold Schabel <[email protected]>
Reviewed-by: Lorenz Brun <[email protected]>
  • Loading branch information
fionera committed Jul 25, 2024
1 parent d7f5993 commit e5e90a8
Show file tree
Hide file tree
Showing 24 changed files with 17,266 additions and 10,408 deletions.
10 changes: 0 additions & 10 deletions BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,6 @@ load("@bazeldnf//:def.bzl", "bazeldnf")
# gazelle:exclude .bazeldnf/**
gazelle(name = "gazelle")

gazelle(
name = "gazelle-update-repos",
args = [
"-from_file=go.mod",
"-to_macro=third_party/go/repositories.bzl%go_repositories",
"-prune",
],
command = "update-repos",
)

# Shortcut for the Go SDK
alias(
name = "go",
Expand Down
90 changes: 73 additions & 17 deletions MODULE.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -18,28 +18,84 @@ single_version_override(
version = "0.0.10",
)

bazel_dep(name = "rules_go", repo_name = "io_bazel_rules_go")
single_version_override(
module_name = "rules_go",
patch_strip = 1,
patches = [
"//third_party/go/patches:rules_go_absolute_embedsrc.patch",
],
version = "0.49.0",
)

bazel_dep(name = "gazelle", repo_name = "bazel_gazelle")
single_version_override(
module_name = "gazelle",
patch_strip = 1,
patches = [
"//third_party/gazelle:add-prepatching.patch",
],
version = "0.37.0",
)

bazel_dep(name = "rules_rust")
single_version_override(
module_name = "rules_rust",
patch_strip = 1,
patches = [
"//third_party:rust-uefi-platform.patch",
"//third_party:rust-prost-nostd.patch",
"//third_party:rust-reproducibility.patch",
],
version = "0.47.1",
)

bazel_dep(name = "zlib")
single_version_override(
module_name = "zlib",
version = "1.3.1",
)

# Fix bazel 7.2 compatibility https://github.com/grpc/grpc-java/issues/11275
# This is coming from the grpc dependency.
# ├───[email protected]
# │ ├───[email protected]
# │ │ └───[email protected]
single_version_override(
module_name = "grpc-java",
version = "1.64.0",
)

bazel_dep(name = "bazel_skylib", version = "1.7.1")
bazel_dep(name = "rules_pkg", version = "1.0.1")
bazel_dep(name = "rules_oci", version = "1.8.0")
bazel_dep(name = "aspect_bazel_lib", version = "2.7.8")
bazel_dep(name = "rules_multirun", version = "0.9.0")
bazel_dep(name = "rules_python", version = "0.34.0")
bazel_dep(name = "rules_cc", version = "0.0.9")
bazel_dep(name = "grpc", version = "1.65.0")
bazel_dep(name = "rules_proto", version = "6.0.2")
bazel_dep(name = "protobuf", version = "27.2")

oci = use_extension("@rules_oci//oci:extensions.bzl", "oci")

# Declare external images you need to pull, for example:
oci.pull(
name = "distroless_base",
# 'latest' is not reproducible, but it's convenient.
# During the build we print a WARNING message that includes recommended 'digest' and 'platforms'
# values which you can use here in place of 'tag' to pin for reproducibility.
digest = "sha256:6c1e34e2f084fe6df17b8bceb1416f1e11af0fcdb1cef11ee4ac8ae127cb507c",
image = "gcr.io/distroless/base",
platforms = [
"linux/amd64",
"linux/arm64/v8",
],
)
# Register our custom CC toolchains. Order matters - more specific toolchains must be registered first.
# (host_cc_toolchain won't care about //build/platforms/linkmode, but musl_host_toolchain won't
# match anything unless its linkmode is set).
register_toolchains("//build/toolchain/musl-host-gcc:musl_host_toolchain")

register_toolchains("//build/toolchain/llvm-efi:efi_k8_toolchain")

register_toolchains("//build/toolchain:host_cc_toolchain")

# Rust Toolchains
register_toolchains("@rust_toolchains//:all")

# Protobuf Toolchains
register_toolchains("@rules_rust//proto/protobuf:default-proto-toolchain")

register_toolchains("//build/rust:prost_efi_toolchain")

include("//:rust.MODULE.bazel")

include("//:go.MODULE.bazel")

# For each oci.pull call, repeat the "name" here to expose them as dependencies.
use_repo(oci, "distroless_base")
include("//:oci.MODULE.bazel")
19,367 changes: 16,644 additions & 2,723 deletions MODULE.bazel.lock

Large diffs are not rendered by default.

163 changes: 0 additions & 163 deletions WORKSPACE
Original file line number Diff line number Diff line change
@@ -1,175 +1,12 @@
workspace(name = "dev_source_monogon")

load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file")
load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository")

# Assert minimum Bazel version

load("@bazel_skylib//lib:versions.bzl", "versions")

versions.check(minimum_bazel_version = "7.0.0")

# Register our custom CC toolchains. Order matters - more specific toolchains must be registered first.
# (host_cc_toolchain won't care about //build/platforms/linkmode, but musl_host_toolchain won't
# match anything unless its linkmode is set).
register_toolchains("//build/toolchain/musl-host-gcc:musl_host_toolchain")

register_toolchains("//build/toolchain/llvm-efi:efi_k8_toolchain")

register_toolchains("//build/toolchain:host_cc_toolchain")

# Go and Gazelle
http_archive(
name = "io_bazel_rules_go",
patch_args = ["-p1"],
patches = [
"//third_party/go/patches:rules_go_absolute_embedsrc.patch",
"//third_party/go/patches:rules_go_gopackagesdriver_anchor_kind.patch",
],
sha256 = "80a98277ad1311dacd837f9b16db62887702e9f1d1c4c9f796d0121a46c8e184",
urls = [
"https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.46.0/rules_go-v0.46.0.zip",
"https://github.com/bazelbuild/rules_go/releases/download/v0.46.0/rules_go-v0.46.0.zip",
],
)

http_archive(
name = "bazel_gazelle",
integrity = "sha256-12v3pg/YsFBEQJDfooN6Tq+YKeEWVhjuNdzspcvfWNU=",
patch_args = ["-p1"],
patches = [
"//third_party/gazelle:add-prepatching.patch",
],
urls = [
"https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.37.0/bazel-gazelle-v0.37.0.tar.gz",
"https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.37.0/bazel-gazelle-v0.37.0.tar.gz",
],
)

load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies")
load("//third_party/go:repositories.bzl", "go_repositories")

# gazelle:repository_macro third_party/go/repositories.bzl%go_repositories
# gazelle:repository go_repository name=dev_source_monogon importpath=source.monogon.dev
go_repositories()

go_rules_dependencies()

go_register_toolchains(
go_version = "1.22.1",
nogo = "@dev_source_monogon//build/analysis:nogo",
)

load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies")

# Load Gazelle-generated local dependencies
gazelle_dependencies()

# Protobuf

http_archive(
name = "rules_proto",
sha256 = "dc3fb206a2cb3441b485eb1e423165b231235a1ea9b031b4433cf7bc1fa460dd",
strip_prefix = "rules_proto-5.3.0-21.7",
urls = [
"https://github.com/bazelbuild/rules_proto/archive/refs/tags/5.3.0-21.7.tar.gz",
],
)

load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")

rules_proto_dependencies()

rules_proto_toolchains()

# GRPC
http_archive(
name = "com_github_grpc_grpc",
integrity = "sha256-Q3BouLd307M52pTTSY8dwgZCrJv6dttDq91SIYaxVCs=",
patch_args = ["-p1"],
patches = ["//third_party/go/patches:grpc_extra_deps.patch"],
strip_prefix = "grpc-1.60.0",
urls = [
"https://github.com/grpc/grpc/archive/v1.60.0.tar.gz",
],
)

load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps")

grpc_deps()

load("@com_github_grpc_grpc//bazel:grpc_extra_deps.bzl", "grpc_extra_deps")

grpc_extra_deps()

# Rust rules
http_archive(
name = "rules_rust",
integrity = "sha256-35cwdTOaqqu4y+aXgIUU2C2PAKMz4+uyJ7/UMIGCmFs=",
patch_args = ["-p1"],
patches = [
"//third_party:rust-uefi-platform.patch",
"//third_party:rust-prost-nostd.patch",
],
urls = ["https://github.com/bazelbuild/rules_rust/releases/download/0.47.1/rules_rust-v0.47.1.tar.gz"],
)

load("@rules_rust//rust:repositories.bzl", "rules_rust_dependencies", "rust_register_toolchains")

rules_rust_dependencies()

rust_register_toolchains(
edition = "2021",
extra_target_triples = [
"x86_64-unknown-uefi",
],
versions = ["1.71.0"],
)

load("@rules_rust//crate_universe:repositories.bzl", "crate_universe_dependencies")

crate_universe_dependencies()

load("@rules_rust//crate_universe:defs.bzl", "crates_repository")

crates_repository(
name = "crate_index",
cargo_lockfile = "//third_party/rust:Cargo.lock",
generate_binaries = True,
lockfile = "//third_party/rust:Cargo.Bazel.lock",
manifests = ["//third_party/rust:Cargo.toml"],
)

load("@crate_index//:defs.bzl", "crate_repositories")

crate_repositories()

crates_repository(
name = "crate_index_efi",
cargo_lockfile = "//third_party/rust_efi:Cargo.lock",
generate_binaries = True,
lockfile = "//third_party/rust_efi:Cargo.Bazel.lock",
manifests = ["//third_party/rust_efi:Cargo.toml"],
supported_platform_triples = [
"x86_64-unknown-linux-gnu",
"x86_64-unknown-uefi",
],
)

load("@crate_index_efi//:defs.bzl", crate_efi_repositories = "crate_repositories")

crate_efi_repositories()

load("@rules_rust//proto/prost:repositories.bzl", "rust_prost_dependencies")

rust_prost_dependencies()

load("@rules_rust//proto/prost:transitive_repositories.bzl", "rust_prost_transitive_repositories")

rust_prost_transitive_repositories()

register_toolchains("//build/rust:prost_efi_toolchain")

# third_party external repositories
load("//third_party/linux:external.bzl", "linux_external")

Expand Down
3 changes: 1 addition & 2 deletions build/ci/jenkins-presubmit.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,6 @@ pipeline {
sh "git clean -fdx -e '/bazel-*'"
sh "JENKINS_NODE_COOKIE=dontKillMe tools/bazel --bazelrc=.bazelrc.ci mod deps --lockfile_mode=update"
sh "JENKINS_NODE_COOKIE=dontKillMe tools/bazel --bazelrc=.bazelrc.ci run //:go -- mod tidy"
sh "JENKINS_NODE_COOKIE=dontKillMe tools/bazel --bazelrc=.bazelrc.ci run //:gazelle-update-repos"
sh "JENKINS_NODE_COOKIE=dontKillMe tools/bazel --bazelrc=.bazelrc.ci run //:gazelle -- update"

script {
Expand All @@ -61,7 +60,7 @@ pipeline {
Please run:
\$ bazel mod deps --lockfile_mode=update
\$ bazel run //:gazelle-update-repos
\$ bazel run //:go -- mod tidy
\$ bazel run //:gazelle -- update
In your git checkout and amend the resulting diff to this changelist.
Expand Down
57 changes: 22 additions & 35 deletions build/print-workspace-status.py
Original file line number Diff line number Diff line change
Expand Up @@ -147,57 +147,44 @@ def parse_tag(tag: str, product: str) -> Optional[Version]:

# Special treatment for Kubernetes, which uses these stamp values in its build
# system. We populate the Kubernetes version from whatever is in
# //third_party/go/repositories.bzl.
def parse_repositories_bzl(path: str) -> dict[str, str]:
# //go.mod.
def parse_go_mod(path: str) -> dict[str, str]:
"""
Shoddily parse a Gazelle-created repositories.bzl into a map of
name->version.
Shoddily parse a go.mod into a map of name->version.
This relies heavily on repositories.bzl being correctly formatted and
This relies heavily on go.mod being correctly formatted and
sorted.
If this breaks, it's probably best to try to use the actual Python parser
to deal with this, eg. by creating a fake environment for the .bzl file to
be parsed.
If this breaks, it's probably best to try to port this to Go
and parse it using golang.org/x/mod/modfile, shell out to
"go mod edit -json", or similar.
"""

# Main parser state: None where we don't expect a version line, set to some
# value otherwise.
name: Optional[str] = None
# Just a copied together regex to find the url followed by a semver.
NAME_VERSION_REGEX = r"([-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*) v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)"

res = {}
for line in open(path):
line = line.strip()
if line == "go_repository(":
name = None
matches = re.findall(NAME_VERSION_REGEX, line)
if not matches:
continue
if line.startswith("name ="):
if name is not None:
raise Exception("parse error in repositories.bzl: repeated name?")
if line.count('"') != 2:
raise Exception(
"parse error in repositories.bzl: invalid name line: " + name
)
name = line.split('"')[1]

[name, version] = matches[0][0].strip().split(" ")

# If we already saw a package, skip it.
if name in res:
continue
if line.startswith("version ="):
if name is None:
raise Exception("parse error in repositories.bzl: version before name")
if line.count('"') != 2:
raise Exception(
"parse error in repositories.bzl: invalid name line: " + name
)
version = line.split('"')[1]
res[name] = version
name = None

res[name] = version

return res


# Parse repositories.bzl.
go_versions = parse_repositories_bzl("third_party/go/repositories.bzl")
# Parse go.mod.
go_versions = parse_go_mod("go.mod")

# Find Kubernetes version.
kubernetes_version: str = go_versions.get("io_k8s_kubernetes")
kubernetes_version: str = go_versions.get("k8s.io/kubernetes")
if kubernetes_version is None:
raise Exception("could not figure out Kubernetes version")
kubernetes_version_parsed = re.match(
Expand Down
Loading

0 comments on commit e5e90a8

Please sign in to comment.