From 96fbbf87c91e0def59bd2ebd3a04ba62c6bfda9c Mon Sep 17 00:00:00 2001 From: Brian Watson Date: Tue, 30 Nov 2021 12:56:30 -0700 Subject: [PATCH] Revert "Revert "Crystalball post-merge map generation"" Also moves config/initializer/crystalball.rb to spec/support to prevent it from running in prod This reverts commit 9a9c68be6e6fda1c300866347f8cd83b5b9201db. Test-plan: - passes cd to edge without failure - generates map successfully in PoC build Change-Id: I67c02ebaea06e11b3a3721aa49217da16e75bd32 Reviewed-on: https://gerrit.instructure.com/c/canvas-lms/+/279848 Reviewed-by: Michael Hargiss Reviewed-by: James Butters Tested-by: Service Cloud Jenkins QA-Review: Brian Watson Product-Review: Brian Watson --- .gitignore | 1 + Gemfile.d/test.rb | 2 + Jenkinsfile.crystalball | 140 ++++++++++++++++++ .../new-jenkins/crystalball_merge_coverage.rb | 52 +++++++ .../library/vars/rspecStage.groovy | 12 +- config/crystalball.yml | 28 ++++ spec/spec_helper.rb | 23 +++ spec/support/crystalball.rb | 133 +++++++++++++++++ 8 files changed, 390 insertions(+), 1 deletion(-) create mode 100644 Jenkinsfile.crystalball create mode 100644 build/new-jenkins/crystalball_merge_coverage.rb create mode 100644 config/crystalball.yml create mode 100644 spec/support/crystalball.rb diff --git a/.gitignore b/.gitignore index 86dabae8b31b2..aadae9521d064 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ /config/*.yml /config/credentials.yml.enc !/config/credentials.test.yml +!/config/crystalball.yml /config/environments/*-local.rb /config/locales/generated/ /config/RAILS6_1 diff --git a/Gemfile.d/test.rb b/Gemfile.d/test.rb index 02198ec4c3799..cef1086ae2e9a 100644 --- a/Gemfile.d/test.rb +++ b/Gemfile.d/test.rb @@ -74,4 +74,6 @@ # performance tools for instrumenting rspec tests gem "stackprof" + + gem "crystalball", "0.7.0", require: false end diff --git a/Jenkinsfile.crystalball b/Jenkinsfile.crystalball new file mode 100644 index 0000000000000..ec2fdc42bf8de --- /dev/null +++ b/Jenkinsfile.crystalball @@ -0,0 +1,140 @@ +#!/usr/bin/env groovy + +/* + * Copyright (C) 2021 - present Instructure, Inc. + * + * This file is part of Canvas. + * + * Canvas is free software: you can redistribute it and/or modify it under + * the terms of the GNU Affero General Public License as published by the Free + * Software Foundation, version 3 of the License. + * + * Canvas is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR + * A PARTICULAR PURPOSE. See the GNU Affero General Public License for more + * details. + * + * You should have received a copy of the GNU Affero General Public License along + * with this program. If not, see . + */ + +library 'canvas-builds-library' +loadLocalLibrary('local-lib', 'build/new-jenkins/library') + +// if the build never starts or gets into a node block, then we +// can never load a file. and a very noisy/confusing error is thrown. +def ignoreBuildNeverStartedError(block) { + try { + block() + } + catch (org.jenkinsci.plugins.workflow.steps.MissingContextVariableException ex) { + if (!ex.message.startsWith('Required context class hudson.FilePath is missing')) { + throw ex + } + else { + echo "ignored MissingContextVariableException: \n${ex.message}" + } + // we can ignore this very noisy error + } +} + +def getMigrationsTag(name) { + (env.GERRIT_REFSPEC.contains('master')) || !migrations.cacheLoadFailed() ? migrations.imageMergeTag(name) : migrations.imagePatchsetTag(name) +} + +def getPatchsetTag() { + (env.GERRIT_REFSPEC.contains('master')) ? "${configuration.buildRegistryPath()}:${env.GERRIT_BRANCH}" : imageTag.patchset() +} + +def getResultsHTMLUrl() { + return "${env.BUILD_URL}/artifact/crystalball_map.yml" +} + +pipeline { + agent { label 'canvas-docker' } + options { + ansiColor('xterm') + timestamps() + } + + environment { + BUILD_REGISTRY_FQDN = configuration.buildRegistryFQDN() + POSTGRES = configuration.postgres() + RUBY = configuration.ruby() // RUBY_VERSION is a reserved keyword for ruby installs + // e.g. canvas-lms:01.123456.78-postgres-12-ruby-2.6 + PATCHSET_TAG = getPatchsetTag() + + CASSANDRA_PREFIX = configuration.buildRegistryPath('cassandra-migrations') + DYNAMODB_PREFIX = configuration.buildRegistryPath('dynamodb-migrations') + POSTGRES_PREFIX = configuration.buildRegistryPath('postgres-migrations') + + IMAGE_CACHE_MERGE_SCOPE = configuration.gerritBranchSanitized() + RSPEC_PROCESSES = 6 + + CASSANDRA_IMAGE_TAG = "$CASSANDRA_PREFIX:$IMAGE_CACHE_MERGE_SCOPE-$RSPEC_PROCESSES" + DYNAMODB_IMAGE_TAG = "$DYNAMODB_PREFIX:$IMAGE_CACHE_MERGE_SCOPE-$RSPEC_PROCESSES" + POSTGRES_IMAGE_TAG = "$POSTGRES_PREFIX:$IMAGE_CACHE_MERGE_SCOPE-$RSPEC_PROCESSES" + } + + stages { + stage('Setup') { + steps { + cleanAndSetup() + } + } + + stage('Parallel Run Tests') { + steps { + script { + def stages = [:] + + distribution.stashBuildScripts() + rspecStage.createDistribution(stages) + + parallel(stages) + } + } + } + } + + post { + always { + script { + ignoreBuildNeverStartedError { + node('master') { + buildSummaryReport.publishReport('Build Summary Report', currentBuild.getResult() == 'SUCCESS' ? 'SUCCESS' : 'FAILURE') + } + } + + copyArtifacts( + filter: 'tmp/crystalball/**', + optional: false, + projectName: env.JOB_NAME, + selector: specific(env.BUILD_NUMBER), + ) + + sh """ + docker-compose run -v \$(pwd)/\$LOCAL_WORKDIR/tmp/crystalball/:/tmp/crystalball \ + -v \$(pwd)/\$LOCAL_WORKDIR/build:/usr/src/app/build \ + --name crystalball-parser \ + web bash -c 'ruby build/new-jenkins/crystalball_merge_coverage.rb /tmp/crystalball/' + """ + + sh 'docker cp crystalball-parser:/usr/src/app/crystalball_map.yml .' + archiveArtifacts allowEmptyArchive: true, artifacts: 'crystalball_map.yml' + + // Only alert on periodic jobs, not ones resulting from manual tests + if (env.GERRIT_EVENT_TYPE != 'comment-added') { + slackSend channel: '#crystalball-noisy', message: "<$env.BUILD_URL/testReport|Latest Crystalball Map Generated> - <${getResultsHTMLUrl()}|Map>" + } + } + } + cleanup { + script { + ignoreBuildNeverStartedError { + libraryScript.execute 'bash/docker-cleanup.sh --allow-failure' + } + } + } + } +} diff --git a/build/new-jenkins/crystalball_merge_coverage.rb b/build/new-jenkins/crystalball_merge_coverage.rb new file mode 100644 index 0000000000000..d16ad87285a6c --- /dev/null +++ b/build/new-jenkins/crystalball_merge_coverage.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +# +# Copyright (C) 2021 - present Instructure, Inc. +# +# This file is part of Canvas. +# +# Canvas is free software: you can redistribute it and/or modify it under +# the terms of the GNU Affero General Public License as published by the Free +# Software Foundation, version 3 of the License. +# +# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +# details. +# +# You should have received a copy of the GNU Affero General Public License along +# with this program. If not, see . +# + +path = "/tmp/crystalball" +map_header = nil +map_body = {} +Dir.glob("#{path}/**/*_map.yml") do |filename| + puts "Looking through #{filename}" + doc = File.read(filename) + (header, body) = doc.split("---").reject(&:empty?) + map_header ||= header + body.split("\n").slice_when { |_before, after| after.include?(":") }.each do |group| + spec = group.shift + changed_files = group + + next if spec.empty? || changed_files.count.zero? + + raise "#{spec} already has entries: #{map_body[spec]}" unless map_body[spec].nil? + + map_body[spec] = changed_files + end +end + +File.open("crystalball_map.yml", "w") do |file| + file << "---" + file << map_header + file << "---" + file << "\n" + map_body.each do |spec, app_files| + file.puts spec + file.puts app_files.join("\n") + end +end + +puts "Crystalball Map Created for #{map_body.keys.count} tests!" diff --git a/build/new-jenkins/library/vars/rspecStage.groovy b/build/new-jenkins/library/vars/rspecStage.groovy index d9cfcd7247166..ce901d430b0a7 100644 --- a/build/new-jenkins/library/vars/rspecStage.groovy +++ b/build/new-jenkins/library/vars/rspecStage.groovy @@ -28,6 +28,7 @@ def createDistribution(nestedStages) { def baseEnvVars = [ "ENABLE_AXE_SELENIUM=${env.ENABLE_AXE_SELENIUM}", + "ENABLE_CRYSTALBALL=${env.ENABLE_CRYSTALBALL}", 'POSTGRES_PASSWORD=sekret', 'SELENIUM_VERSION=3.141.59-20201119', "RSPECQ_ENABLED=${env.RSPECQ_ENABLED}" @@ -103,6 +104,14 @@ def tearDownNode(prefix) { sh 'build/new-jenkins/docker-copy-files.sh /usr/src/app/log/results tmp/rspec_results canvas_ --allow-error --clean-dir' sh "build/new-jenkins/docker-copy-files.sh /usr/src/app/log/spec_failures/ tmp/spec_failures/$prefix canvas_ --allow-error --clean-dir" + // Don't generate map pre-merge + if (env.ENABLE_CRYSTALBALL == '1' && env.RSPECQ_ENABLED != '1') { + sh 'build/new-jenkins/docker-copy-files.sh /usr/src/app/log/results/crystalball_results tmp/crystalball canvas_ --allow-error --clean-dir' + sh 'ls tmp/crystalball' + sh 'ls -R' + archiveArtifacts allowEmptyArchive: true, artifacts: 'tmp/crystalball/**/*' + } + if (configuration.getBoolean('upload-docker-logs', 'false')) { sh "docker ps -aq | xargs -I{} -n1 -P1 docker logs --timestamps --details {} 2>&1 > tmp/docker-${prefix}-${CI_NODE_INDEX}.log" archiveArtifacts(artifacts: "tmp/docker-${prefix}-${CI_NODE_INDEX}.log") @@ -149,6 +158,7 @@ def runRspecqSuite() { return } sh(script: 'docker-compose exec -T -e ENABLE_AXE_SELENIUM \ + -e ENABLE_CRYSTALBALL \ -e RSPECQ_ENABLED \ -e SENTRY_DSN \ -e RSPECQ_UPDATE_TIMINGS \ @@ -181,7 +191,7 @@ def runRspecqSuite() { def runLegacySuite() { try { - sh(script: 'docker-compose exec -T -e RSPEC_PROCESSES -e ENABLE_AXE_SELENIUM canvas bash -c \'build/new-jenkins/rspec-with-retries.sh\'', label: 'Run Tests') + sh(script: 'docker-compose exec -T -e RSPEC_PROCESSES -e ENABLE_AXE_SELENIUM -e ENABLE_CRYSTALBALL canvas bash -c \'build/new-jenkins/rspec-with-retries.sh\'', label: 'Run Tests') } catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e) { if (e.causes[0] instanceof org.jenkinsci.plugins.workflow.steps.TimeoutStepExecution.ExceededTimeout) { /* groovylint-disable-next-line GStringExpressionWithinString */ diff --git a/config/crystalball.yml b/config/crystalball.yml new file mode 100644 index 0000000000000..b39f599b57af5 --- /dev/null +++ b/config/crystalball.yml @@ -0,0 +1,28 @@ +# Custom path to your execution map. Can be file or folder. Default: `tmp/crystalball_data.yml` +execution_map_path: './crystalball_map.yml' +# Maximum amount of examples which will be run automatically. Default: no limit. +# If prediction size is over the limit Crystalball will prune prediction to fit the limit. +# examples_limit: 1 +# +# Custom prediction builder class to use. Default: 'Crystalball::RSpec::StandardPredictionBuilder' +prediction_builder_class_name: 'Crystalball::RSpec::StandardPredictionBuilder' +# # +# Set of requires. Usually used to require custom predictor class file. Default: [] +requires: + - './spec/support/crystalball.rb' +# +# Custom map expiration period in seconds. 0 = Disable expiration check. Default: 86400 (1 day) +map_expiration_period : 0 +# +# Custom RSpec runner class to use. Default: 'Crystalball::RSpec::Runner' +runner_class_name: 'Crystalball::RSpec::DryRunner' +# +# File path to save dry-run prediction results +dry_run_output_file_path: './crystalball_spec_list.txt' +# +# "From" value for `git diff` command. default: 'HEAD' +diff_from: 'HEAD' +# +# "To" value for `git diff` command. default: nil +diff_to: + diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 48209ed283cb1..56e0732667fa9 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -32,6 +32,7 @@ require "securerandom" require "tmpdir" +require "crystalball" ENV["RAILS_ENV"] = "test" require_relative "../config/environment" @@ -156,6 +157,28 @@ def view_assigns end end +# Don't do map generation in pre-merge, which runs rspecq +if ENV["ENABLE_CRYSTALBALL"] == "1" && ENV["RSPECQ_ENABLED"] != "1" + Crystalball::MapGenerator.start! do |config| + config.register Crystalball::MapGenerator::CoverageStrategy.new + config.map_storage_path = "log/results/crystalball_results/#{ENV.fetch("PARALLEL_INDEX", "0")}_map.yml" + end +end + +module Crystalball + class MapGenerator + class CoverageStrategy + def call(example_map, example) + puts "Calling Coverage Strategy for #{example.inspect}" + before = Coverage.peek_result + yield example_map, example + after = Coverage.peek_result + example_map.push(*execution_detector.detect(before, after)) + end + end + end +end + module RSpec::Rails module ViewExampleGroup module ExampleMethods diff --git a/spec/support/crystalball.rb b/spec/support/crystalball.rb new file mode 100644 index 0000000000000..c833f90f3da34 --- /dev/null +++ b/spec/support/crystalball.rb @@ -0,0 +1,133 @@ +# frozen_string_literal: true + +# +# Copyright (C) 2021 - present Instructure, Inc. +# +# This file is part of Canvas. +# +# Canvas is free software: you can redistribute it and/or modify it under +# the terms of the GNU Affero General Public License as published by the Free +# Software Foundation, version 3 of the License. +# +# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +# details. +# +# You should have received a copy of the GNU Affero General Public License along +# with this program. If not, see . + +require "rspec/core" +require "crystalball/rspec/prediction_builder" +require "crystalball/rspec/filtering" +require "crystalball/rspec/prediction_pruning" + +module Crystalball + module RSpec + # Our custom RSpec runner to generate and save predictions to a file, i.e. "dry-run" + # TODO: make DryRunner class NOT inherit ::RSpec::Core::Runner because we don"t need it + class DryRunner < ::RSpec::Core::Runner + include PredictionPruning + + class << self + def run(args, err = $stderr, out = $stdout) + return config["runner_class"].run(args, err, out) unless config["runner_class"] == self + + Crystalball.log :info, "Crystalball starts to glow..." + prediction = build_prediction + dry_run(prediction) if args.include?("--dry-run") + + Crystalball.log :debug, "Prediction: #{prediction.first(5).join(" ")}#{"..." if prediction.size > 5}" + Crystalball.log :info, "Starting RSpec." + + super(args + prediction, err, out) + end + + def dry_run(prediction) + prediction_file_path = config["dry_run_output_file_path"] + File.write(prediction_file_path, prediction.to_a.join(",")) + Crystalball.log :info, "Saved RSpec prediction to #{prediction_file_path}" + exit # rubocop:disable Rails/Exit + end + + def reset! + self.prediction_builder = nil + self.config = nil + end + + def prepare + config["runner_class"].load_execution_map + end + + def prediction_builder + @prediction_builder ||= config["prediction_builder_class"].new(config) + end + + def config + @config ||= begin + config_src = if config_file + require "yaml" + YAML.safe_load(config_file.read) + else + {} + end + + Crystalball::RSpec::Runner::Configuration.new(config_src) + end + end + + protected + + def load_execution_map + check_map + prediction_builder.execution_map + end + + private + + attr_writer :config, :prediction_builder + + def config_file + file = Pathname.new(ENV.fetch("CRYSTALBALL_CONFIG", "crystalball.yml")) + file = Pathname.new("config/crystalball.yml") unless file.exist? + file.exist? ? file : nil + end + + def build_prediction + check_map + prune_prediction_to_limit(prediction_builder.prediction.sort_by(&:length)) + end + + def check_map + Crystalball.log :warn, "Maps are outdated!" if prediction_builder.expired_map? + end + end + + def setup(err, out) + configure(err, out) + @configuration.load_spec_files + + Filtering.remove_unnecessary_filters(@configuration, @options.options[:files_or_directories_to_run]) + + if reconfiguration_needed? + Crystalball.log :warn, "Prediction examples size #{@world.example_count} is over the limit (#{examples_limit})" + Crystalball.log :warn, "Prediction is pruned to fit the limit!" + + reconfigure_to_limit + @configuration.load_spec_files + end + + @world.announce_filters + end + + # Backward compatibility for RSpec < 3.7 + def configure(err, out) + @configuration.error_stream = err + @configuration.output_stream = out if @configuration.output_stream == $stdout + @options.configure(@configuration) + end + end + end +end + +require "crystalball/rspec/runner/configuration"