Support build with Bazel (apache#4865)

* Support build with Bazel, fixing tests to make them capable of running in concurrency and hermetic.
llIlll · Aug 23, 2022 · ec8a93d · ec8a93d
1 parent 92d10ba
commit ec8a93d
Show file tree

Hide file tree

Showing 105 changed files with 1,880 additions and 243 deletions.
diff --git a/.bazelrc b/.bazelrc
@@ -0,0 +1,73 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+startup --host_jvm_args=-Xmx2g
+
+run --color=yes
+
+build --color=yes
+build --enable_platform_specific_config
+
+test --action_env=TEST_TMPDIR=/tmp
+
+test  --experimental_strict_java_deps=warn
+build --experimental_strict_java_deps=warn
+
+
+# This .bazelrc file contains all of the flags required for the provided
+# toolchain with Remote Build Execution.
+# Note your WORKSPACE must contain an rbe_autoconfig target with
+# name="rbe_default" to use these flags as-is.
+
+# Depending on how many machines are in the remote execution instance, setting
+# this higher can make builds faster by allowing more jobs to run in parallel.
+# Setting it too high can result in jobs that timeout, however, while waiting
+# for a remote machine to execute them.
+build:remote --jobs=150
+
+# Set several flags related to specifying the platform, toolchain and java
+# properties.
+# These flags should only be used as is for the rbe-ubuntu16-04 container
+# and need to be adapted to work with other toolchain containers.
+build:remote --java_runtime_version=rbe_jdk
+build:remote --tool_java_runtime_version=rbe_jdk
+build:remote --extra_toolchains=@rbe_default//java:all
+
+build:remote --crosstool_top=@rbe_default//cc:toolchain
+build:remote --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
+# Platform flags:
+# The toolchain container used for execution is defined in the target indicated
+# by "extra_execution_platforms", "host_platform" and "platforms".
+# More about platforms: https://docs.bazel.build/versions/master/platforms.html
+build:remote --extra_toolchains=@rbe_default//config:cc-toolchain
+build:remote --extra_execution_platforms=@rbe_default//config:platform
+build:remote --host_platform=@rbe_default//config:platform
+build:remote --platforms=@rbe_default//config:platform
+
+# Starting with Bazel 0.27.0 strategies do not need to be explicitly
+# defined. See https://github.com/bazelbuild/bazel/issues/7480
+build:remote --define=EXECUTOR=remote
+
+# Enable remote execution so actions are performed on the remote systems.
+build:remote --remote_executor=grpcs://remote.buildbuddy.io
+
+# Enforce stricter environment rules, which eliminates some non-hermetic
+# behavior and therefore improves both the remote cache hit rate and the
+# correctness and repeatability of the build.
+build:remote --incompatible_strict_action_env=true
+
+# Set a higher timeout value, just in case.
+build:remote --remote_timeout=3600
diff --git a/.bazelversion b/.bazelversion
@@ -0,0 +1 @@
+5.2.0
diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml
@@ -0,0 +1,22 @@
+name: Build and Run Tests By Bazel
+on:
+  pull_request_target:
+    types: [opened, reopened, synchronize]
+  push:
+    branches:
+      - master
+      - develop
+      - bazel
+jobs:
+  build:
+    name: "Java (${{ matrix.os }})"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-18.04, ubuntu-20.04, ubuntu-22.04]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Build
+        run: bazel build --config=remote --remote_header=x-buildbuddy-api-key=${{ secrets.BUILD_BUDDY_API_KEY }} //...
+      - name: Run Tests
+        run: bazel test --config=remote --remote_header=x-buildbuddy-api-key=${{ secrets.BUILD_BUDDY_API_KEY }} //...
diff --git a/.github/workflows/build.yaml → .github/workflows/maven.yaml b/.github/workflows/build.yaml → .github/workflows/maven.yaml
@@ -1,4 +1,4 @@
-name: Build
+name: Build and Run Tests By Maven
 on:
   pull_request:
     types: [opened, reopened, synchronize]

diff --git a/.gitignore b/.gitignore
@@ -13,3 +13,7 @@ devenv
 .DS_Store
 localbin
 nohup.out
+bazel-out
+bazel-bin
+bazel-rocketmq
+bazel-testlogs
diff --git a/.licenserc.yaml b/.licenserc.yaml
@@ -42,6 +42,7 @@ header:
     - 'distribution/LICENSE-BIN'
     - 'distribution/NOTICE-BIN'
     - 'distribution/conf/rmq-proxy.json'
+    - '.bazelversion'
 
 
   comment: on-failure
diff --git a/BUILD.bazel b/BUILD.bazel
@@ -0,0 +1,46 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+load("@bazel_toolchains//rules/exec_properties:exec_properties.bzl", "create_rbe_exec_properties_dict")
+
+platform(
+	name = "custom_platform",
+    # Inherit from the platform target generated by 'rbe_configs_gen' assuming the generated configs
+    # were imported as a Bazel external repository named 'rbe_default'. If you extracted the
+    # generated configs elsewhere in your source repository, replace the following with the label
+    # to the 'platform' target in the generated configs.
+	parents = ["@rbe_default//config:platform"],
+    # Example custom execution property instructing RBE to use e2-standard-2 GCE VMs.
+	exec_properties = create_rbe_exec_properties_dict(
+		container_image = "ubuntu:latest",
+	),
+)
+
+java_library(
+    name = "test_deps",
+    visibility = ["//visibility:public"],
+    exports = [
+        "@maven//:junit_junit",
+        "@maven//:org_assertj_assertj_core",
+        "@maven//:org_hamcrest_hamcrest_library",
+        "@maven//:org_mockito_mockito_core",
+        "@maven//:org_hamcrest_hamcrest_core",
+        "@maven//:ch_qos_logback_logback_classic",
+        "@maven//:org_awaitility_awaitility",
+        "@maven//:org_openjdk_jmh_jmh_core",
+        "@maven//:org_openjdk_jmh_jmh_generator_annprocess",
+    ],
+)
diff --git a/WORKSPACE b/WORKSPACE
@@ -0,0 +1,128 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+RULES_JVM_EXTERNAL_TAG = "4.2"
+
+RULES_JVM_EXTERNAL_SHA = "cd1a77b7b02e8e008439ca76fd34f5b07aecb8c752961f9640dea15e9e5ba1ca"
+
+http_archive(
+    name = "rules_jvm_external",
+    sha256 = RULES_JVM_EXTERNAL_SHA,
+    strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG,
+    url = "https://github.com/bazelbuild/rules_jvm_external/archive/%s.zip" % RULES_JVM_EXTERNAL_TAG,
+)
+
+load("@rules_jvm_external//:repositories.bzl", "rules_jvm_external_deps")
+
+rules_jvm_external_deps()
+
+load("@rules_jvm_external//:setup.bzl", "rules_jvm_external_setup")
+
+rules_jvm_external_setup()
+
+load("@rules_jvm_external//:defs.bzl", "maven_install")
+
+maven_install(
+    artifacts = [
+        "junit:junit:4.13.2",
+        "com.alibaba:fastjson:1.2.76",
+        "org.hamcrest:hamcrest-library:1.3",
+        "io.netty:netty-all:4.1.65.Final",
+        "org.slf4j:slf4j-api:1.7.7",
+        "org.assertj:assertj-core:3.22.0",
+        "org.mockito:mockito-core:3.10.0",
+        "com.github.luben:zstd-jni:1.5.2-2",
+        "org.lz4:lz4-java:1.8.0",
+        "commons-validator:commons-validator:1.7",
+        "org.apache.commons:commons-lang3:3.4",
+        "org.hamcrest:hamcrest-core:1.3",
+      # "io.openmessaging.storage:dledger:0.2.4",
+        "net.java.dev.jna:jna:4.2.2",
+        "ch.qos.logback:logback-classic:1.2.10",
+        "ch.qos.logback:logback-core:1.2.10",
+        "io.opentracing:opentracing-api:0.33.0",
+        "io.opentracing:opentracing-mock:0.33.0",
+        "commons-collections:commons-collections:3.2.2",
+        "org.awaitility:awaitility:4.1.0",
+        "commons-cli:commons-cli:1.4",
+        "com.google.guava:guava:31.0.1-jre",
+        "org.yaml:snakeyaml:1.30",
+        "commons-codec:commons-codec:1.13",
+        "commons-io:commons-io:2.7",
+        "log4j:log4j:1.2.17",
+        "com.google.truth:truth:0.30",
+        "org.bouncycastle:bcpkix-jdk15on:1.69",
+        "com.google.code.gson:gson:2.8.9",
+        "com.googlecode.concurrentlinkedhashmap:concurrentlinkedhashmap-lru:1.4.2",
+        "org.apache.rocketmq:rocketmq-proto:2.0.0",
+        "com.google.protobuf:protobuf-java:3.20.1",
+        "com.google.protobuf:protobuf-java-util:3.20.1",
+        "com.conversantmedia:disruptor:1.2.10",
+        "javax.annotation:javax.annotation-api:1.3.2",
+        "com.google.code.findbugs:jsr305:3.0.2",
+        "org.checkerframework:checker-qual:3.12.0",
+        "org.reflections:reflections:0.9.11",
+        "org.openjdk.jmh:jmh-core:1.19",
+        "org.openjdk.jmh:jmh-generator-annprocess:1.19",
+        "com.github.ben-manes.caffeine:caffeine:2.9.3",
+        "io.grpc:grpc-services:1.47.0",
+        "io.grpc:grpc-netty-shaded:1.47.0",
+        "io.grpc:grpc-context:1.47.0",
+        "io.grpc:grpc-stub:1.47.0",
+        "io.grpc:grpc-api:1.47.0",
+        "io.grpc:grpc-testing:1.47.0",
+    ],
+    fetch_sources = True,
+    repositories = [
+        # Private repositories are supported through HTTP Basic auth
+        "https://repo1.maven.org/maven2",
+    ],
+)
+
+http_archive(
+    name = "io_buildbuddy_buildbuddy_toolchain",
+    sha256 = "a2a5cccec251211e2221b1587af2ce43c36d32a42f5d881737db3b546a536510",
+    strip_prefix = "buildbuddy-toolchain-829c8a574f706de5c96c54ca310f139f4acda7dd",
+    urls = ["https://github.com/buildbuddy-io/buildbuddy-toolchain/archive/829c8a574f706de5c96c54ca310f139f4acda7dd.tar.gz"],
+)
+
+load("@io_buildbuddy_buildbuddy_toolchain//:deps.bzl", "buildbuddy_deps")
+
+buildbuddy_deps()
+
+load("@io_buildbuddy_buildbuddy_toolchain//:rules.bzl", "buildbuddy")
+
+buildbuddy(name = "buildbuddy_toolchain")
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+http_archive(
+    name = "rbe_default",
+    # sha256 = "c0d428774cbe70d477e1d07581d863f8dbff4ba6a66d20502d7118354a814bea",
+    urls = ["https://storage.googleapis.com/rbe-toolchain/bazel-configs/rbe-ubuntu1604/latest/rbe_default.tar"],
+)
+
+http_archive(
+	name = "bazel_toolchains",
+	urls = ["https://github.com/bazelbuild/bazel-toolchains/archive/dac71231098d891e5c4b74a2078fe9343feef510.tar.gz"],
+	strip_prefix = "bazel-toolchains-dac71231098d891e5c4b74a2078fe9343feef510",
+	sha256 = "56d5370eb99559b4c74f334f81bc8a298f728bd16d5a4333c865c2ad10fae3bc",
+)
+
+load("@bazel_toolchains//repositories:repositories.bzl", bazel_toolchains_repositories = "repositories")
+bazel_toolchains_repositories()
diff --git a/acl/BUILD.bazel b/acl/BUILD.bazel
@@ -0,0 +1,73 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+load("//bazel:GenTestRules.bzl", "GenTestRules")
+
+java_library(
+    name = "acl",
+    srcs = glob(["src/main/java/**/*.java"]),
+    visibility = ["//visibility:public"],
+    deps = [
+        "//common",
+        "//remoting",
+        "//logging", 
+        "//srvutil",
+        "@maven//:org_apache_commons_commons_lang3",
+        "@maven//:commons_validator_commons_validator",
+        "@maven//:com_github_luben_zstd_jni",
+        "@maven//:org_lz4_lz4_java",
+        "@maven//:com_alibaba_fastjson",
+        "@maven//:io_netty_netty_all",
+        "@maven//:org_yaml_snakeyaml",
+        "@maven//:commons_codec_commons_codec",
+        "@maven//:org_apache_rocketmq_rocketmq_proto",
+        "@maven//:com_google_protobuf_protobuf_java",
+        "@maven//:com_google_guava_guava",
+    ],
+)
+
+java_library(
+    name = "tests",
+    srcs = glob(["src/test/java/**/*.java"]),
+    visibility = ["//visibility:public"],
+    deps = [
+        ":acl",
+        "//common",
+        "//remoting",
+        "//:test_deps",
+        "@maven//:org_apache_commons_commons_lang3", 
+        "@maven//:io_netty_netty_all", 
+        "@maven//:org_yaml_snakeyaml",
+        "@maven//:commons_codec_commons_codec",   
+        "@maven//:com_alibaba_fastjson",           
+    ],
+    resources = glob(["src/test/resources/**/*.yml"])
+)
+
+GenTestRules(
+    name = "GeneratedTestRules",
+    test_files = glob(["src/test/java/**/*Test.java"]),
+    deps = [
+        ":tests",
+    ],
+    # The following tests are not hermetic. Fix them later.
+    exclude_tests = [
+        "src/test/java/org/apache/rocketmq/acl/plain/PlainAccessControlFlowTest",
+        "src/test/java/org/apache/rocketmq/acl/plain/PlainPermissionManagerTest",
+        "src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest",
+        "src/test/java/org/apache/rocketmq/acl/common/AclUtilsTest",
+    ],
+)
diff --git a/acl/src/main/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyFactory.java b/acl/src/main/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyFactory.java
@@ -50,18 +50,18 @@ public RemoteAddressStrategy getRemoteAddressStrategy(String remoteAddr) {
                 String[] strArray = StringUtils.split(remoteAddr, ":");
                 String last = strArray[strArray.length - 1];
                 if (!last.startsWith("{")) {
-                    throw new AclException(String.format("MultipleRemoteAddressStrategy netaddress examine scope Exception netaddress", remoteAddr));
+                    throw new AclException(String.format("MultipleRemoteAddressStrategy netaddress examine scope Exception netaddress: %s", remoteAddr));
                 }
                 return new MultipleRemoteAddressStrategy(AclUtils.getAddresses(remoteAddr, last));
             } else {
                 String[] strArray = StringUtils.split(remoteAddr, ".");
                 // However a right IP String provided by user,it always can be divided into 4 parts by '.'.
                 if (strArray.length < 4) {
-                    throw new AclException(String.format("MultipleRemoteAddressStrategy has got a/some wrong format IP(s) ", remoteAddr));
+                    throw new AclException(String.format("MultipleRemoteAddressStrategy has got a/some wrong format IP(s): %s ", remoteAddr));
                 }
                 String lastStr = strArray[strArray.length - 1];
                 if (!lastStr.startsWith("{")) {
-                    throw new AclException(String.format("MultipleRemoteAddressStrategy netaddress examine scope Exception netaddress", remoteAddr));
+                    throw new AclException(String.format("MultipleRemoteAddressStrategy netaddress examine scope Exception netaddress: %s", remoteAddr));
                 }
                 return new MultipleRemoteAddressStrategy(AclUtils.getAddresses(remoteAddr, lastStr));
             }