Bug 1703483 - Add required CI jobs for using clang main as a testing …

…toolchain for firefox build. r=mhentges,glandium

1. Add task to get and build clang from the main branch.
2. Using clang main toolchain we build on a daily basis linux64 firefox, this tasks also automatically triggers the fetch and build of clang from main branch since we don't cache it.

Differential Revision: https://phabricator.services.mozilla.com/D111063

.cron.yml

@@ -162,6 +162,17 @@ jobs:
       when:
           - {hour: 10, minute: 0}
 
+    - name: linux64-bp-clang-trunk
+      job:
+          type: decision-task
+          treeherder-symbol: linux64-bp-clang-trunk
+          target-tasks-method: linux64_bp_clang_trunk
+      run-on-projects:
+          - mozilla-central
+          - mozilla-release
+      when:
+          - {hour: 2, minute: 0}
+
     - name: updatebot-cron-job
       job:
           type: decision-task

browser/config/mozconfigs/linux64/plain-opt-trunk

@@ -0,0 +1,3 @@
+# Use `-gcc-toolchain` extra argument until `Bug 1704766` lands.
+export CXX="$MOZ_FETCHES_DIR/clang/bin/clang++ -gcc-toolchain $MOZ_FETCHES_DIR/clang"
+export CC="$MOZ_FETCHES_DIR/clang/bin/clang -gcc-toolchain $MOZ_FETCHES_DIR/clang"

build/build-clang/clang-trunk-linux64.json

@@ -0,0 +1,23 @@
+{
+  "stages": "2",
+  "build_libcxx": true,
+  "build_wasm": true,
+  "build_type": "Release",
+  "assertions": false,
+  "python_path": "/usr/bin/python2.7",
+  "gcc_dir": "{MOZ_FETCHES_DIR}/gcc",
+  "cc": "{MOZ_FETCHES_DIR}/gcc/bin/gcc",
+  "cxx": "{MOZ_FETCHES_DIR}/gcc/bin/g++",
+  "as": "{MOZ_FETCHES_DIR}/gcc/bin/gcc",
+  "wasi-sysroot": "{MOZ_FETCHES_DIR}/wasi-sysroot",
+  "patches": [
+    "static-llvm-symbolizer_clang_12.patch",
+    "find_symbolizer_linux_clang_10.patch",
+    "android-mangling-error_clang_12.patch",
+    "unpoison-thread-stacks_clang_10.patch",
+    "downgrade-mangling-error_clang_12.patch",
+    "revert-llvmorg-12-init-7827-g2a078c307204.patch",
+    "loosen-msvc-detection.patch",
+    "clang-trunk-missing-define.patch"
+  ]
+}

build/build-clang/clang-trunk-missing-define.patch

@@ -0,0 +1,19 @@
+diff --git a/clang-tools-extra/clangd/quality/CompletionModel.cmake b/clang-tools-extra/clangd/quality/CompletionModel.cmake
+index 41bc2ed1890b..e8f5345da84e 100644
+--- a/clang-tools-extra/clangd/quality/CompletionModel.cmake
++++ b/clang-tools-extra/clangd/quality/CompletionModel.cmake
+@@ -1,11 +1,11 @@
+-# Run the Completion Model Codegenerator on the model present in the 
++# Run the Completion Model Codegenerator on the model present in the
+ # ${model} directory.
+-# Produces a pair of files called ${filename}.h and  ${filename}.cpp in the 
++# Produces a pair of files called ${filename}.h and  ${filename}.cpp in the
+ # ${CMAKE_CURRENT_BINARY_DIR}. The generated header
+ # will define a C++ class called ${cpp_class} - which may be a
+ # namespace-qualified class name.
+ function(gen_decision_forest model filename cpp_class)
+-  set(model_compiler ${LLVM_EXTERNAL_CLANG_TOOLS_EXTRA_SOURCE_DIR}/clangd/quality/CompletionModelCodegen.py)
++  set(model_compiler ${CMAKE_SOURCE_DIR}/../clang-tools-extra/clangd/quality/CompletionModelCodegen.py)
+
+   set(output_dir ${CMAKE_CURRENT_BINARY_DIR})
+   set(header_file ${output_dir}/${filename}.h)

taskcluster/ci/build/linux.yml

@@ -89,6 +89,45 @@ linux64-plain/opt:
     optimization:
         skip-unless-expanded: null
 
+linux64-plain-clang-trunk/opt:
+    description: "Linux64 Opt Plain with clang trunk"
+    index:
+        product: firefox
+        job-name: linux64-plain-opt-ct
+    treeherder:
+        platform: linux64/opt
+        symbol: Bp-clang-trunk
+        tier: 2
+    worker-type: b-linux
+    worker:
+        docker-image: {in-tree: debian10-amd64-build}
+        max-run-time: 5400
+        env:
+            PERFHERDER_EXTRA_OPTIONS: plain-clang-trunk
+    run:
+        using: mozharness
+        actions: [build]
+        config:
+            - builds/releng_base_firefox.py
+            - builds/releng_base_linux_64_builds.py
+        script: mozharness/scripts/fx_desktop_build.py
+        tooltool-downloads: public
+        extra-config:
+            disable_package_metrics: true
+        mozconfig-variant: plain-opt-trunk
+    fetches:
+        toolchain:
+            - linux64-binutils
+            - linux64-clang-trunk
+            - linux64-rust
+            - linux64-nasm
+            - linux64-node
+            - linux64-cbindgen
+            - linux64-dump-syms
+            - linux64-toolchain-sysroot
+    optimization:
+        skip-unless-expanded: null
+
 linux64-fuzzing/debug:
     description: "Linux64 Fuzzing Debug"
     index:

taskcluster/ci/fetch/toolchains.yml

@@ -441,6 +441,15 @@ clang-12:
         repo: https://github.com/llvm/llvm-project
         revision: 31001be371e8f2c74470e727e54503fb2aabec8b
 
+clang-trunk:
+    description: clang main branch source code
+    attributes:
+        cached_task: false
+    fetch:
+        type: git
+        repo: https://github.com/llvm/llvm-project
+        branch: main
+
 rust-1.47.0-dev:
     description: Rust 1.47.0-dev source code
     fetch:

taskcluster/ci/toolchain/clang.yml

@@ -147,6 +147,29 @@ linux64-clang-11:
             - linux64-toolchain-sysroot
             - wasi-sysroot-11
 
+linux64-clang-trunk:
+    description: "Clang trunk toolchain build"
+    attributes:
+        cached_task: false
+    treeherder:
+        symbol: TL(clang-trunk)
+    run:
+        using: toolchain-script
+        script: build-clang.sh
+        arguments:
+            - 'build/build-clang/clang-trunk-linux64.json'
+        resources:
+            - 'build/build-clang/clang-trunk-linux64.json'
+        toolchain-artifact: public/build/clang.tar.zst
+    fetches:
+        fetch:
+            - clang-trunk
+        toolchain:
+            - linux64-binutils
+            - linux64-gcc-7
+            - linux64-toolchain-sysroot
+            - wasi-sysroot
+
 linux64-clang-11-android-cross:
     description: "Clang 11 toolchain build"
     treeherder:

taskcluster/taskgraph/target_tasks.py

@@ -1040,6 +1040,13 @@ def target_tasks_coverity_full(full_task_graph, parameters, graph_config):
     return ["source-test-coverity-coverity-full-analysis"]
 
 
+# Run build linux64-plain-clang-trunk/opt on mozilla-central/release
+@_target_task("linux64_bp_clang_trunk")
+def target_tasks_build_linux64_clang_trunk(full_task_graph, parameters, graph_config):
+    """Select tasks required to run the build of linux64 build plain with clang trunk"""
+    return ["build-linux64-plain-clang-trunk/opt"]
+
+
 # Run Updatebot's cron job 4 times daily.
 @_target_task("updatebot_cron")
 def target_tasks_updatebot_cron(full_task_graph, parameters, graph_config):

taskcluster/taskgraph/transforms/cached_tasks.py

@@ -55,7 +55,10 @@ def cache_task(config, tasks):
 
     digests = {}
     for task in config.kind_dependencies_tasks.values():
-        if "cached_task" in task.attributes:
+        if (
+            "cached_task" in task.attributes
+            and task.attributes["cached_task"] is not False
+        ):
             digests[task.label] = format_task_digest(task.attributes["cached_task"])
 
     for task in order_tasks(config, tasks):

taskcluster/taskgraph/transforms/fetch.py

@@ -20,6 +20,7 @@
     Optional,
     Required,
     Extra,
+    Any,
 )
 
 from mozpack import path as mozpath
@@ -142,7 +143,9 @@ def make_task(config, jobs):
             "attributes": attributes,
             "name": name,
             "description": job["description"],
-            "expires-after": expires,
+            "expires-after": "2 days"
+            if attributes.get("cached_task") is False
+            else expires,
             "label": "fetch-%s" % name,
             "run-on-projects": [],
             "treeherder": {
@@ -292,7 +295,7 @@ def create_fetch_url_task(config, name, fetch):
     "git",
     schema={
         Required("repo"): text_type,
-        Required("revision"): text_type,
+        Required(Any("revision", "branch")): text_type,
         Optional("artifact-name"): text_type,
         Optional("path-prefix"): text_type,
         # ssh-key is a taskcluster secret path (e.g. project/civet/github-deploy-key)
@@ -310,16 +313,26 @@ def create_git_fetch_task(config, name, fetch):
     if not artifact_name:
         artifact_name = "{}.tar.zst".format(path_prefix)
 
-    if not re.match(r"[0-9a-fA-F]{40}", fetch["revision"]):
-        raise Exception('Revision is not a sha1 in fetch task "{}"'.format(name))
+    if "revision" in fetch and "branch" in fetch:
+        raise Exception("revision and branch cannot be used in the same context")
+
+    revision_or_branch = None
+
+    if "revision" in fetch:
+        revision_or_branch = fetch["revision"]
+        if not re.match(r"[0-9a-fA-F]{40}", fetch["revision"]):
+            raise Exception('Revision is not a sha1 in fetch task "{}"'.format(name))
+    else:
+        # we are sure we are dealing with a branch
+        revision_or_branch = fetch["branch"]
 
     args = [
         "/builds/worker/bin/fetch-content",
         "git-checkout-archive",
         "--path-prefix",
         path_prefix,
         fetch["repo"],
-        fetch["revision"],
+        revision_or_branch,
         "/builds/worker/artifacts/%s" % artifact_name,
     ]
 
@@ -331,7 +344,7 @@ def create_git_fetch_task(config, name, fetch):
     return {
         "command": args,
         "artifact_name": artifact_name,
-        "digest_data": [fetch["revision"], path_prefix, artifact_name],
+        "digest_data": [revision_or_branch, path_prefix, artifact_name],
         "secret": ssh_key,
     }
 

taskcluster/taskgraph/transforms/job/toolchain.py

@@ -151,12 +151,14 @@ def docker_worker_toolchain(config, job, taskdesc):
     if "toolchain-alias" in run:
         attributes["toolchain-alias"] = run.pop("toolchain-alias")
 
-    if not taskgraph.fast:
+    digest_data = get_digest_data(config, run, taskdesc)
+
+    if job.get("attributes", {}).get("cached_task") is not False and not taskgraph.fast:
         name = taskdesc["label"].replace("{}-".format(config.kind), "", 1)
         taskdesc["cache"] = {
             "type": CACHE_TYPE,
             "name": name,
-            "digest-data": get_digest_data(config, run, taskdesc),
+            "digest-data": digest_data,
         }
 
     run["using"] = "run-task"
@@ -211,12 +213,14 @@ def generic_worker_toolchain(config, job, taskdesc):
     if "toolchain-alias" in run:
         attributes["toolchain-alias"] = run.pop("toolchain-alias")
 
-    if not taskgraph.fast:
+    digest_data = get_digest_data(config, run, taskdesc)
+
+    if job.get("attributes", {}).get("cached_task") is not False and not taskgraph.fast:
         name = taskdesc["label"].replace("{}-".format(config.kind), "", 1)
         taskdesc["cache"] = {
             "type": CACHE_TYPE,
             "name": name,
-            "digest-data": get_digest_data(config, run, taskdesc),
+            "digest-data": digest_data,
         }
 
     run["using"] = "run-task"

taskcluster/taskgraph/util/cached_tasks.py

@@ -38,6 +38,10 @@ def add_optimization(
         task.
     :type digest_data: list of bytes or None
     """
+    cached_task = taskdesc.get("attributes", {}).get("cached_task")
+    if cached_task is False:
+        return
+
     if (digest is None) == (digest_data is None):
         raise Exception("Must pass exactly one of `digest` and `digest_data`.")
     if digest is None: