Merge branch 'ali-golang-script-extract-build-proto' into 'master'

[IDX-2479] Golang script to extract Bazel BEP proto and export to Honeycomb

Add Go binary that takes the build event stream protobuf and exports the data to Honeycomb. This will be helpful to identify failing and flaky tests across multiple commits.

Demo: https://ui.honeycomb.io/dfinity/datasets/bazel/result/svvVUEBEcGZ

![image](/uploads/a704c932a7e5284b66bebc08557576a7/image.png)

This change introduce Gazelle and Rules Go to the repository.

To manage external dependencies
- Add them to `go.mod` [e.g. manually or with `go get ...`]
- Run `bazel run //:gazelle-update-repos` to update `deps.bzl`

To add new Go code
- Run `bazel run //:gazelle` to auto-generate and format new `BUILD.bazel` files.

These MR also vendors in BEP protos.
* These are copies of the protos available in the Bazel repo, but also provide rules for Go definitions
* Modified the protobuf import paths to include the `bazel/proto` prefix instead of just `proto/`
* The proto library from the Bazel repo has restricted visibility [except for the Java definition which we don't want]

See merge request dfinity-lab/public/ic!9040

.bazelrc

@@ -31,7 +31,9 @@ build --remote_timeout=30s # Defauilt is 60s.
 build:ci --remote_timeout=5m # Defauilt is 60s.
 
 
+# Does not produce valid JSON. See https://github.com/bazelbuild/bazel/issues/14209
 build --execution_log_json_file=bazel-build-log.json
+build --build_event_binary_file=bazel-bep.pb
 
 build --bes_results_url=https://dash.buildfarm.dfinity.systems/invocation/
 build --bes_backend=bes.buildfarm.dfinity.systems

.gitignore

@@ -116,6 +116,8 @@ scalability/results/
 # Bazel outdir dirs
 /bazel-*
 bazel-build-log.json
+bazel-bep.pb
+bazel-bep.txt
 user.bazelrc
 
 # Visual Studio Code

BUILD.bazel

@@ -1,3 +1,5 @@
+load("@bazel_gazelle//:def.bzl", "gazelle")
+
 package(default_visibility = ["//visibility:public"])
 
 exports_files([
@@ -27,3 +29,20 @@ alias(
     name = "rustfmt",
     actual = "@rules_rust//:rustfmt",
 )
+
+# See https://github.com/bazelbuild/bazel-gazelle#running-gazelle-with-bazel
+# gazelle:prefix github.com/dfinity/ic
+# gazelle:proto disable
+gazelle(
+    name = "gazelle",
+)
+
+gazelle(
+    name = "gazelle-update-repos",
+    args = [
+        "-from_file=go.mod",
+        "-to_macro=go_deps.bzl%go_dependencies",
+        "-prune",
+    ],
+    command = "update-repos",
+)

WORKSPACE.bazel

@@ -9,6 +9,33 @@ load("//third_party/wabt-rs-0.10.0:repository.bzl", "wabt_rs_repository")
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository", "new_git_repository")
 
+http_archive(
+    name = "io_bazel_rules_go",
+    sha256 = "099a9fb96a376ccbbb7d291ed4ecbdfd42f6bc822ab77ae6f1b5cb9e914e94fa",
+    urls = [
+        "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.35.0/rules_go-v0.35.0.zip",
+        "https://github.com/bazelbuild/rules_go/releases/download/v0.35.0/rules_go-v0.35.0.zip",
+    ],
+)
+
+http_archive(
+    name = "bazel_gazelle",
+    sha256 = "448e37e0dbf61d6fa8f00aaa12d191745e14f07c31cabfa731f0c8e8a4f41b97",
+    urls = [
+        "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.28.0/bazel-gazelle-v0.28.0.tar.gz",
+        "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.28.0/bazel-gazelle-v0.28.0.tar.gz",
+    ],
+)
+
+load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies")
+load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies")
+
+go_rules_dependencies()
+
+go_register_toolchains(go_version = "1.19.3")
+
+gazelle_dependencies(go_repository_default_config = "//:WORKSPACE.bazel")
+
 http_archive(
     name = "bazel_skylib",
     sha256 = "f7be3474d42aae265405a592bb7da8e171919d74c16f082a5457840f06054728",
@@ -55,6 +82,10 @@ http_archive(
 )
 
 load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")
+load("//:go_deps.bzl", "go_dependencies")
+
+# gazelle:repository_macro go_deps.bzl%go_dependencies
+go_dependencies()
 
 rules_proto_dependencies()
 

bazel/exporter/BUILD.bazel

@@ -0,0 +1,20 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+    name = "exporter_lib",
+    srcs = ["main.go"],
+    importpath = "github.com/dfinity/ic/bazel/exporter",
+    visibility = ["//visibility:private"],
+    deps = [
+        "//bazel/proto:build_event_stream_go_proto",
+        "@com_github_golang_protobuf//proto:go_default_library",
+        "@com_github_honeycombio_beeline_go//:beeline-go",
+        "@org_golang_google_protobuf//encoding/protojson",
+    ],
+)
+
+go_binary(
+    name = "exporter",
+    embed = [":exporter_lib"],
+    visibility = ["//visibility:public"],
+)

bazel/exporter/main.go

@@ -0,0 +1,174 @@
+// Bazel invoked with --build_event_binary_file outputs a series of delimited build event stream protobuf messages to a file.
+// This binary parse that file and output the data to Honeycomb.
+// Example:
+//   bazel run //bazel/exporter:exporter -- -f (git rev-parse --show-toplevel)/bazel/exporter/testdata/flaky-bep.pb
+package main
+
+import (
+	"bufio"
+	"context"
+	"encoding/binary"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"os"
+
+	"github.com/dfinity/ic/proto/build_event_stream"
+	"github.com/golang/protobuf/proto"
+	beeline "github.com/honeycombio/beeline-go"
+	"google.golang.org/protobuf/encoding/protojson"
+)
+
+// Expect multiple proto messages in uvarint delimited format.
+func ReadDelimitedProtoMessage(br *bufio.Reader) ([]byte, error) {
+	size, err := binary.ReadUvarint(br)
+	if err != nil {
+		return nil, err
+	}
+
+	msg := make([]byte, size)
+	if _, err := io.ReadFull(br, msg); err != nil {
+		return nil, fmt.Errorf("error reading protobuf", err)
+	}
+
+	return msg, nil
+}
+
+func loadEnvVars() map[string]interface{} {
+	want := [...]string{"CD_ENV",
+		"CI_COMMIT_AUTHOR",
+		"CI_COMMIT_SHA",
+		"CI_COMMIT_TAG",
+		"CI_COMMIT_TIMESTAMP",
+		"CI_CONCURRENT_ID",
+		"CI_CONCURRENT_PROJECT_ID",
+		"CI_ENVIRONMENT_NAME",
+		"CI_ENVIRONMENT_SLUG",
+		"CI_EXTERNAL_PULL_REQUEST_IID",
+		"CI_EXTERNAL_PULL_REQUEST_SOURCE_BRANCH_NAME",
+		"CI_EXTERNAL_PULL_REQUEST_SOURCE_BRANCH_SHA",
+		"CI_JOB_ID",
+		"CI_JOB_IMAGE",
+		"CI_JOB_MANUAL",
+		"CI_JOB_NAME",
+		"CI_JOB_STAGE",
+		"CI_JOB_STATUS",
+		"CI_JOB_URL",
+		"CI_NODE_INDEX",
+		"CI_NODE_TOTAL",
+		"CI_PIPELINE_ID",
+		"CI_PIPELINE_SOURCE",
+		"CI_RUNNER_DESCRIPTION",
+		"CI_RUNNER_ID",
+		"CI_RUNNER_TAGS",
+		"DEPLOY_FLAVOR",
+		"USER_ID",
+		"USER_LOGIN",
+		"SCHEDULE_NAME",
+		"TESTNET",
+		"STEP_START",
+		"PIPELINE_START_TIME",
+		"job_status",
+		"DISKIMG_BRANCH",
+		"CI_MERGE_REQUEST_APPROVED",
+		"CI_MERGE_REQUEST_ASSIGNEES",
+		"CI_MERGE_REQUEST_ID",
+		"CI_MERGE_REQUEST_IID",
+		"CI_MERGE_REQUEST_LABELS",
+		"CI_MERGE_REQUEST_MILESTONE",
+		"CI_MERGE_REQUEST_PROJECT_ID",
+		"CI_MERGE_REQUEST_PROJECT_PATH",
+		"CI_MERGE_REQUEST_PROJECT_URL",
+		"CI_MERGE_REQUEST_REF_PATH",
+		"CI_MERGE_REQUEST_SOURCE_BRANCH_NAME",
+		"CI_MERGE_REQUEST_SOURCE_BRANCH_SHA",
+		"CI_MERGE_REQUEST_SOURCE_PROJECT_ID",
+		"CI_MERGE_REQUEST_SOURCE_PROJECT_PATH",
+		"CI_MERGE_REQUEST_SOURCE_PROJECT_URL",
+		"CI_MERGE_REQUEST_TARGET_BRANCH_NAME",
+		"CI_MERGE_REQUEST_TARGET_BRANCH_SHA",
+		"CI_MERGE_REQUEST_TITLE",
+		"CI_MERGE_REQUEST_EVENT_TYPE",
+		"CI_MERGE_REQUEST_DIFF_ID",
+		"CI_MERGE_REQUEST_DIFF_BASE_SHA",
+	}
+
+	env_vars := make(map[string]interface{})
+	for _, w := range want {
+		env_vars[w] = os.Getenv(w)
+	}
+	return env_vars
+}
+
+func envVarOrDie(name string) string {
+	ans := os.Getenv(name)
+	if ans == "" {
+		log.Fatalln("Could not load env var ", name)
+	}
+	return ans
+}
+
+func main() {
+	filename := flag.String("f", "", "Bazel build events log protobuff file")
+	debug := flag.Bool("n", false, "Debug mode: Output all the proto in text json text form")
+	flag.Parse()
+
+	beeline.Init(beeline.Config{
+		WriteKey:    envVarOrDie("HONEYCOMB_API_TOKEN"),
+		Dataset:     "bazel",
+		ServiceName: "exporter",
+	})
+	defer beeline.Close()
+
+	pbfile, err := os.Open(*filename)
+	if err != nil {
+		log.Fatalln(err)
+	}
+	br := bufio.NewReader(pbfile)
+
+	envVars := loadEnvVars()
+	log.Println("Reading file", pbfile.Name())
+	cnt := 0
+	for ; ; cnt++ {
+		msg, err := ReadDelimitedProtoMessage(br)
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			log.Fatalln("failed to read next proto message", err)
+		}
+
+		event := &build_event_stream.BuildEvent{}
+		if err := proto.Unmarshal(msg, event); err != nil {
+			log.Fatalln("Failed to unmarshal", err)
+		}
+		if *debug {
+			fmt.Println(protojson.Format(event))
+		}
+
+		// Proto message is oneof many types. Check if it's a test summary message. Otherwise skip it.
+		summary := event.GetTestSummary()
+		if summary == nil {
+			continue
+		}
+
+		// Marhsal the protobuf to Json format. This does things like converts proto enums to their string representation.
+		b, err := protojson.Marshal(event)
+		if err != nil {
+			log.Fatalln("failed to marshal protobuf to json:", err)
+		}
+
+		jsonMap := make(map[string]interface{})
+		if err := json.Unmarshal(b, &jsonMap); err != nil {
+			log.Fatalln("failed to unmarshal json bytes to map: ", err)
+		}
+
+		spanCtx, eventSpan := beeline.StartSpan(context.Background(), "export_event")
+		beeline.AddField(spanCtx, "event", jsonMap)
+		beeline.AddField(spanCtx, "gitlab", envVars)
+		eventSpan.Send()
+	}
+
+	log.Printf("Processed %d protos", cnt)
+}

bazel/proto/BUILD.bazel

@@ -0,0 +1,82 @@
+load("@rules_proto//proto:defs.bzl", "proto_library")
+load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library")
+
+package(default_visibility = ["//visibility:public"])
+
+proto_library(
+    name = "build_event_stream_proto",
+    srcs = ["build_event_stream.proto"],
+    deps = [
+        ":command_line_proto",
+        ":failure_details_proto",
+        ":invocation_policy_proto",
+        "@com_google_protobuf//:duration_proto",
+        "@com_google_protobuf//:timestamp_proto",
+    ],
+)
+
+proto_library(
+    name = "command_line_proto",
+    srcs = ["command_line.proto"],
+    deps = [
+        ":option_filters_proto",
+    ],
+)
+
+proto_library(
+    name = "failure_details_proto",
+    srcs = [
+        "failure_details.proto",
+    ],
+    deps = [
+        "@com_google_protobuf//:descriptor_proto",
+    ],
+)
+
+proto_library(
+    name = "option_filters_proto",
+    srcs = ["option_filters.proto"],
+)
+
+proto_library(
+    name = "invocation_policy_proto",
+    srcs = ["invocation_policy.proto"],
+)
+
+go_proto_library(
+    name = "command_line_go_proto",
+    importpath = "github.com/dfinity/ic/proto/command_line",
+    proto = ":command_line_proto",
+    deps = [
+        ":option_filters_go_proto",
+    ],
+)
+
+go_proto_library(
+    name = "option_filters_go_proto",
+    importpath = "github.com/dfinity/ic/proto/options",
+    proto = ":option_filters_proto",
+)
+
+go_proto_library(
+    name = "failure_details_go_proto",
+    importpath = "github.com/dfinity/ic/proto/failure_details",
+    proto = ":failure_details_proto",
+)
+
+go_proto_library(
+    name = "invocation_policy_go_proto",
+    importpath = "github.com/dfinity/ic/proto/blaze.invocation_policy",
+    proto = ":invocation_policy_proto",
+)
+
+go_proto_library(
+    name = "build_event_stream_go_proto",
+    importpath = "github.com/dfinity/ic/proto/build_event_stream",
+    proto = ":build_event_stream_proto",
+    deps = [
+        ":command_line_go_proto",
+        ":failure_details_go_proto",
+        ":invocation_policy_go_proto",
+    ],
+)

bazel/proto/README.md

@@ -0,0 +1,10 @@
+To update the protos
+1. Copy the relavant protos from https://github.com/buildbuddy-io/buildbuddy/tree/master/proto
+1. Edit the import fields to prefix the import paths with `bazel/` e.g. `proto/foo.proto` -> `bazel/proto/foo.proto`
+
+The source of truth for these protobufs files lives in the Bazel repo: https://github.com/bazelbuild/bazel
+
+Unfortunately, it's difficult to either vendor or reference directly from the third party sources.
+1. In the Bazel repo, the Java protobuf definitions are marked public, but their proto_library are private. 
+1. The Buildbuddry repo includes targets to generate typescript protos, a dependency we don't want to include into our workspace.
+1. Furthermore, Gazelle does not seem to know how to import and generate build files for those protobufs definitions.