From fa4988e42e0a52582951b10ab1b26ac25642f836 Mon Sep 17 00:00:00 2001 From: Aki Sasaki Date: Mon, 29 Mar 2021 20:58:50 +0000 Subject: [PATCH] Bug 1701608 - remove perma-busted python dependency generator. r=releng-reviewers,jmaher I have manually pinned the requirements, and partials are working. We'll lose a way to auto-update our dependencies, but a) it's perma-busted, so realistically we're just losing a busted task, and b) mhentges is working on a way to auto-update pinned dependencies in-tree. Differential Revision: https://phabricator.services.mozilla.com/D110148 --- .cron.yml | 13 -- taskcluster/ci/docker-image/kind.yml | 2 - .../ci/python-dependency-update/kind.yml | 45 ------ .../python-dependency-update/Dockerfile | 34 ----- .../docker/python-dependency-update/README.md | 15 -- .../python-dependency-update/requirements.in | 3 - .../python-dependency-update/requirements.txt | 14 -- .../docker/python-dependency-update/runme.sh | 61 -------- .../scripts/update_pipfiles.sh | 134 ------------------ .../docker/python-dependency-update/setup.sh | 38 ----- taskcluster/docs/kinds.rst | 5 - taskcluster/taskgraph/target_tasks.py | 11 -- 12 files changed, 375 deletions(-) delete mode 100644 taskcluster/ci/python-dependency-update/kind.yml delete mode 100644 taskcluster/docker/python-dependency-update/Dockerfile delete mode 100644 taskcluster/docker/python-dependency-update/README.md delete mode 100644 taskcluster/docker/python-dependency-update/requirements.in delete mode 100644 taskcluster/docker/python-dependency-update/requirements.txt delete mode 100755 taskcluster/docker/python-dependency-update/runme.sh delete mode 100755 taskcluster/docker/python-dependency-update/scripts/update_pipfiles.sh delete mode 100755 taskcluster/docker/python-dependency-update/setup.sh diff --git a/.cron.yml b/.cron.yml index d104953e303d8..29749b422b4fa 100644 --- a/.cron.yml +++ b/.cron.yml @@ -239,19 +239,6 @@ jobs: - {weekday: 'Monday', hour: 10, minute: 0} - {weekday: 'Thursday', hour: 10, minute: 0} - - name: python-dependency-update - job: - type: decision-task - treeherder-symbol: Nfile - target-tasks-method: python_dependency_update - run-on-projects: - - mozilla-central - when: - by-project: - # No default branch - mozilla-central: - - {weekday: 'Monday', hour: 10, minute: 0} - - name: raptor-tp6m job: type: decision-task diff --git a/taskcluster/ci/docker-image/kind.yml b/taskcluster/ci/docker-image/kind.yml index f4892965e911b..44640aa5b7850 100644 --- a/taskcluster/ci/docker-image/kind.yml +++ b/taskcluster/ci/docker-image/kind.yml @@ -146,8 +146,6 @@ jobs: parent: push-to-try periodic-updates: symbol: I(file) - python-dependency-update: - symbol: I(pip) firefox-snap: symbol: I(snap) firefox-flatpak: diff --git a/taskcluster/ci/python-dependency-update/kind.yml b/taskcluster/ci/python-dependency-update/kind.yml deleted file mode 100644 index 80e9471313ea6..0000000000000 --- a/taskcluster/ci/python-dependency-update/kind.yml +++ /dev/null @@ -1,45 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. ---- -loader: taskgraph.loader.transform:loader - -transforms: - - taskgraph.transforms.python_update:transforms - - taskgraph.transforms.task:transforms - - -job-defaults: - name: python_dependency_update - description: Update requirements.txt - run-on-projects: [] - treeherder: - kind: build - platform: linux64/opt - tier: 1 - worker-type: b-linux - worker: - implementation: docker-worker - os: linux - docker-image: {in-tree: python-dependency-update} - taskcluster-proxy: true - max-run-time: 2400 - command: - - /runme.sh - artifacts: - - name: 'public/build/requirements.txt.diff' - path: '/home/worker/artifacts/requirements.txt.diff' - type: file - scopes: - - secrets:get:project/releng/gecko/build/level-{level}/arc-phabricator-token - - -jobs: - funsize-update-generator: - treeherder: - symbol: pydep(fug) - worker: - env: - REQUIREMENTS_FILE: "taskcluster/docker/funsize-update-generator/requirements.in" - PYTHON3: "1" - REVIEWERS: "sfraser, bhearsum" diff --git a/taskcluster/docker/python-dependency-update/Dockerfile b/taskcluster/docker/python-dependency-update/Dockerfile deleted file mode 100644 index c9ff57f0f13b0..0000000000000 --- a/taskcluster/docker/python-dependency-update/Dockerfile +++ /dev/null @@ -1,34 +0,0 @@ -FROM ubuntu:bionic -MAINTAINER Ben Hearsum - -# Required software -ENV DEBIAN_FRONTEND noninteractive - -# %include python/mozbuild/mozbuild/action/tooltool.py -ADD topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /setup/tooltool.py - -# %include testing/mozharness/external_tools/robustcheckout.py -ADD topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py - -# %include taskcluster/docker/recipes/hgrc -COPY topsrcdir/taskcluster/docker/recipes/hgrc /etc/mercurial/hgrc.d/mozilla.rc - -# %include taskcluster/docker/recipes/install-mercurial.sh -ADD topsrcdir/taskcluster/docker/recipes/install-mercurial.sh /setup/install-mercurial.sh - -ADD setup.sh /setup/setup.sh -ADD requirements.txt /setup/requirements.txt - -RUN cd /setup && ./setup.sh - -COPY runme.sh / -COPY scripts/* /home/worker/scripts/ - -ENV HOME /home/worker -ENV SHELL /bin/bash -ENV USER worker -ENV LOGNAME worker -ENV LC_ALL C.UTF-8 -ENV LANG C.UTF-8 - -CMD ["/runme.sh"] diff --git a/taskcluster/docker/python-dependency-update/README.md b/taskcluster/docker/python-dependency-update/README.md deleted file mode 100644 index ab560623ee09f..0000000000000 --- a/taskcluster/docker/python-dependency-update/README.md +++ /dev/null @@ -1,15 +0,0 @@ - -==Python Dependency Updates== - -This docker image contains the necessary dependencies and scripts to update -in-tree requirement.txt produced by `pip-compile --generate-hashes`, produce a -diff, and submit it to Phabricator. - - -==Quick Start== - -```sh -docker build -t python-dependency-update --no-cache --rm . - -docker run -e PYTHON3="1" -e BRANCH="mozilla-central" -e REQUIREMENTS_FILE="taskcluster/docker/funsize-update-generator/requirements.in" python-dependency-update -``` diff --git a/taskcluster/docker/python-dependency-update/requirements.in b/taskcluster/docker/python-dependency-update/requirements.in deleted file mode 100644 index d07311d8a7ffc..0000000000000 --- a/taskcluster/docker/python-dependency-update/requirements.in +++ /dev/null @@ -1,3 +0,0 @@ -# bug 1698453 - pip-tools 6.0.0 breaks pip-compile due to missing -# importlib.metadata plus no --index or --no-index -pip-tools<6 diff --git a/taskcluster/docker/python-dependency-update/requirements.txt b/taskcluster/docker/python-dependency-update/requirements.txt deleted file mode 100644 index 2fd451ba4445e..0000000000000 --- a/taskcluster/docker/python-dependency-update/requirements.txt +++ /dev/null @@ -1,14 +0,0 @@ -# SHA1:e0576f98505bf01e96b0dec209728de4a2bcfb65 -# -# This file is autogenerated by pip-compile-multi -# To update, run: -# -# pip-compile-multi -# -click==7.1.2 - # via pip-tools -pip-tools==5.5.0 - # via -r ./requirements.in - -# The following packages are considered to be unsafe in a requirements file: -# pip diff --git a/taskcluster/docker/python-dependency-update/runme.sh b/taskcluster/docker/python-dependency-update/runme.sh deleted file mode 100755 index 395ec0cd64dea..0000000000000 --- a/taskcluster/docker/python-dependency-update/runme.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash - -set -xe - -# Things to be set by task definition. -# -b branch -# -f requirements_file -# -3 use python3 - - -test "${BRANCH}" -test "${REQUIREMENTS_FILE}" - -PIP_ARG="-2" -if [ -n "${PYTHON3}" ]; then - PIP_ARG="-3" -fi - -export ARTIFACTS_DIR="/home/worker/artifacts" -mkdir -p "$ARTIFACTS_DIR" - -# duplicate the functionality of taskcluster-lib-urls, but in bash.. -queue_base="$TASKCLUSTER_ROOT_URL/api/queue/v1" - -# Get Arcanist API token - -if [ -n "${TASK_ID}" ] -then - curl --location --retry 10 --retry-delay 10 -o /home/worker/task.json "$queue_base/task/$TASK_ID" - ARC_SECRET=$(jq -r '.scopes[] | select(contains ("arc-phabricator-token"))' /home/worker/task.json | awk -F: '{print $3}') -fi -if [ -n "${ARC_SECRET}" ] && getent hosts taskcluster -then - set +x # Don't echo these - secrets_url="${TASKCLUSTER_PROXY_URL}/api/secrets/v1/secret/${ARC_SECRET}" - SECRET=$(curl "${secrets_url}") - TOKEN=$(echo "${SECRET}" | jq -r '.secret.token') -elif [ -n "${ARC_TOKEN}" ] # Allow for local testing. -then - TOKEN="${ARC_TOKEN}" -fi - -if [ -n "${TOKEN}" ] -then - cat >"${HOME}/.arcrc" <&2 - usage - exit 12 -fi - -# Must supply a code branch to work with. -if [ "${BRANCH}" == "" ]; then - echo "Error: You must specify a branch with -b branchname." >&2 - usage - exit 13 -fi - -if [ "${REPODIR}" == "" ]; then - REPODIR="${BASEDIR}/$(basename "${BRANCH}")" -fi - -if [ "${BRANCH}" == "mozilla-central" ]; then - HGREPO="https://${HGHOST}/${BRANCH}" -elif [[ "${BRANCH}" == mozilla-* ]]; then - HGREPO="https://${HGHOST}/releases/${BRANCH}" -else - HGREPO="https://${HGHOST}/projects/${BRANCH}" -fi - -clone_repo - -requirements_basefile="$(basename "${REQUIREMENTS_FILE}")" -requirements_dir="$(dirname "${REQUIREMENTS_FILE}")" -update_requirements "${requirements_dir}" "${requirements_basefile}" -requirements_newfile="${requirements_basefile%%.in}.txt" -DIFF_ARTIFACT="${ARTIFACTS_DIR}/${requirements_newfile}.diff" - -echo "INFO: diffing old/new ${requirements_newfile} into ${DIFF_ARTIFACT}" -${HG} -R "${REPODIR}" diff "${BASEDIR}/${BRANCH}/${requirements_dir}/${requirements_newfile}" | tee "${DIFF_ARTIFACT}" - -COMMIT_MESSAGE="No Bug, ${requirements_dir}/${requirements_newfile} pip-update." - -if ${HG} -R "${REPODIR}" commit -u "${COMMIT_AUTHOR}" -m "${COMMIT_MESSAGE}" -then - ${HG} -R "${REPODIR}" out - push_repo -fi - -echo "All done" diff --git a/taskcluster/docker/python-dependency-update/setup.sh b/taskcluster/docker/python-dependency-update/setup.sh deleted file mode 100755 index 69684a2390932..0000000000000 --- a/taskcluster/docker/python-dependency-update/setup.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -set -ve - -tooltool_fetch() { - cat >manifest.tt - python2.7 /setup/tooltool.py fetch - rm manifest.tt -} - -useradd -d /home/worker -s /bin/bash -m worker - -apt-get update -q -apt-get install -y --no-install-recommends \ - arcanist \ - curl \ - gcc \ - jq \ - libdpkg-perl \ - liblzma-dev \ - python \ - python-dev \ - python-pip \ - python3.8 \ - python3.8-dev \ - python3-setuptools \ - python3-wheel \ - python3-pip - -apt-get clean - -. install-mercurial.sh -pip install -r requirements.txt - -rm -rf /setup diff --git a/taskcluster/docs/kinds.rst b/taskcluster/docs/kinds.rst index 48866b93b3b56..3a88eec7b02a1 100644 --- a/taskcluster/docs/kinds.rst +++ b/taskcluster/docs/kinds.rst @@ -580,11 +580,6 @@ repo-update Repo-Update tasks are tasks that perform some action on the project repo itself, in order to update its state in some way. -python-dependency-update ------------------------- -Python-dependency-update runs `pip-compile --generate-hashes` against the specified `requirements.in` and -submits patches to Phabricator. - partials -------- Partials takes the complete.mar files produced in previous tasks and generates partial diff --git a/taskcluster/taskgraph/target_tasks.py b/taskcluster/taskgraph/target_tasks.py index 5048dd05f4381..7c583deb6a506 100644 --- a/taskcluster/taskgraph/target_tasks.py +++ b/taskcluster/taskgraph/target_tasks.py @@ -1054,17 +1054,6 @@ def target_tasks_chromium_update(full_task_graph, parameters, graph_config): ] -@_target_task("python_dependency_update") -def target_tasks_python_update(full_task_graph, parameters, graph_config): - """Select the set of tasks required to perform nightly in-tree pipfile updates""" - - def filter(task): - # For now any task in the repo-update kind is ok - return task.kind in ["python-dependency-update"] - - return [l for l, t in six.iteritems(full_task_graph.tasks) if filter(t)] - - @_target_task("file_update") def target_tasks_file_update(full_task_graph, parameters, graph_config): """Select the set of tasks required to perform nightly in-tree file updates"""