Skip to content

Commit

Permalink
TINY-10143: Reworked - Remote testing integration (tinymce#9263)
Browse files Browse the repository at this point in the history 
* TINY-10143: Bedrock remote testing

* Non-edge testing skips

* Skip edge-breaking tests. Have been previously documented

Documented skipped tests in TINY-10480

* Update failed tests

* jUnit test results

* Move methods to waluigi library

* Use `Edge` in lambdatest instead
  • Loading branch information
@jscasca
jscasca authored Dec 21, 2023
1 parent de877de commit b6fcc36
Show file tree
Hide file tree
Showing 38 changed files with 2,731 additions and 916 deletions.
27 changes: 21 additions & 6 deletions Gruntfile.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,23 +72,26 @@ const bedrockHeadless = (tests, browser, auto) => {
}
};

const bedrockBrowser = (tests, browserName, osName, bucket, buckets, chunk, auto) => {
const bedrockBrowser = (tests, browserName, osName, bucket, buckets, chunk, remote, auto, opts) => {
const name = opts.name ? opts.name : `${browserName}-${osName}`;
if (tests.length === 0) {
return {};
} else {
return {
browser: {
...bedrockDefaults,
overallTimeout: 1200000,
name: `${browserName}-${osName}`,
overallTimeout: 3600000,
name: name,
browser: browserName,
testfiles: testFolders(tests, auto),
bucket: bucket,
buckets: buckets,
chunk: chunk,
remote: remote,

// we have a few tests that don't play nicely when combined together in the monorepo
retries: 3
retries: 3,
...opts
}
};
}
Expand Down Expand Up @@ -138,6 +141,18 @@ module.exports = function (grunt) {
const activeBrowser = grunt.option('bedrock-browser') || 'chrome-headless';
const headlessBrowser = activeBrowser.endsWith("-headless") ? activeBrowser : 'chrome-headless';
const activeOs = grunt.option('bedrock-os') || 'tests';

const remote = grunt.option('remote');

const bedrockOpts = (grunt, availableOpts) => {
return availableOpts.reduce((opts, opt) => {
const current = grunt.option(opt);
if (current) opts[opt] = current;
return opts;
}, {});
};

const opts = bedrockOpts(grunt, ['name', 'username', 'accesskey', 'sishDomain', 'devicefarmArn', 'devicefarmRegion', 'platformName', 'browserVersion']);
const gruntConfig = {
shell: {
tsc: { command: 'yarn -s tsc' },
Expand All @@ -147,11 +162,11 @@ module.exports = function (grunt) {
},
'bedrock-auto': {
...bedrockHeadless(headlessTests, headlessBrowser, true),
...bedrockBrowser(browserTests, activeBrowser, activeOs, bucket, buckets, chunk, true)
...bedrockBrowser(browserTests, activeBrowser, activeOs, bucket, buckets, chunk, remote, true, opts)
},
'bedrock-manual': {
...bedrockHeadless(headlessTests, headlessBrowser, false),
...bedrockBrowser(browserTests, activeBrowser, activeOs, bucket, buckets, chunk, false)
...bedrockBrowser(browserTests, activeBrowser, activeOs, bucket, buckets, chunk, remote, false, opts)
}
};

Expand Down
230 changes: 120 additions & 110 deletions Jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,45 +3,76 @@

standardProperties()

def runTests(String name, String bedrockCommand, Boolean runAll) {
// Clean out the old XML files before running tests, since we junit import *.XML files
dir('scratch') {
if (isUnix()) {
sh "rm -f *.xml"
} else {
bat "del *.xml"
}
}

def command = runAll ? bedrockCommand + ' --ignore-lerna-changed=true' : bedrockCommand
def testStatus = exec(script: command, returnStatus: true)

echo "Writing JUnit results for ${name} on node: $NODE_NAME"
def runBedrockTest(String name, String command, Boolean runAll, int retry = 0, int timeout = 0) {
def bedrockCmd = command + (runAll ? " --ignore-lerna-changed=true" : "")
echo "Running Bedrock cmd: ${command}"
def testStatus = sh(script: command, returnStatus: true)
junit allowEmptyResults: true, testResults: 'scratch/TEST-*.xml'

// If the tests failed (exit code 4) then just mark it as unstable
if (testStatus == 4) {
unstable("Tests failed for ${name}")
} else if (testStatus != 0) {
error("Unexpected error running tests for ${name} so passing failure as exit code")
if (retry > 0) {
echo "Running retry [${retry}] after [${timeout}]"
sleep(timeout)
runBedrockTest(name, command, runAll, retry - 1, timeout)
} else {
archiveArtifacts artifacts: 'scratch/TEST-*.xml', onlyIfSuccessful: false
error("Unexpected error in ${name} ")
}
}
}

def runBrowserTests(String name, String browser, String os, Integer bucket, Integer buckets, Boolean runAll) {
def runHeadlessTests(Boolean runAll) {
def bedrockCmd = "yarn grunt headless-auto"
runBedrockTest('headless', bedrockCmd, runAll)
}

def runRemoteTests(String name, String browser, String provider, String platform, String arn, String bucket, String buckets, Boolean runAll, int retry = 0, int timeout = 0) {
def awsOpts = " --sishDomain=sish.osu.tiny.work --devicefarmArn=${arn}"
def platformName = platform != null ? " --platformName='${platform}'" : ""
def bedrockCommand =
"yarn grunt browser-auto" +
" --chunk=400" +
" --bedrock-os=" + os +
" --bedrock-browser=" + browser +
" --bucket=" + bucket +
" --buckets=" + buckets;

runTests(name, bedrockCommand, runAll);
"yarn browser-test" +
" --chunk=400" +
" --bedrock-browser=" + browser +
" --remote=" + provider +
" --bucket=" + bucket +
" --buckets=" + buckets +
" --name=" + name +
"${provider == 'aws' ? awsOpts : ''}" +
"${platformName}"
runBedrockTest(name, bedrockCommand, runAll, retry, timeout)
}

def runHeadlessTests(Boolean runAll) {
def bedrockCommand = "yarn grunt headless-auto";
runTests("chrome-headless", bedrockCommand, runAll);
def runTestPod(String name, String browser, String provider, String platform, String bucket, String buckets, Boolean runAll) {
return {
tinyPods.node([
resourceRequestCpu: '2',
resourceRequestMemory: '4Gi',
resourceRequestEphemeralStorage: '16Gi',
resourceLimitCpu: '7.5',
resourceLimitMemory: '4Gi',
resourceLimitEphemeralStorage: '16Gi',
]) {
if (provider == 'aws') {
stage('Tunnel') {
bedrockRemoteTools.tinyWorkSishTunnel()
}
}

stage('Test') {
yarnInstall()
sh 'yarn ci'
grunt('list-changed-browser')
bedrockRemoteTools.withRemoteCreds(provider) {
int retry = provider == 'lambdatest' ? 1 : 0
withCredentials([string(credentialsId: 'devicefarm-testgridarn', variable: 'DF_ARN')]) {
runRemoteTests(name, browser, provider, platform, DF_ARN, bucket, buckets, runAll, retry, 180)
}
}
}
}
}
}

def gitMerge(String primaryBranch) {
Expand All @@ -51,115 +82,94 @@ def gitMerge(String primaryBranch) {
}
}

def props

timestamps {
// TinyMCE builds need more CPU and RAM (especially eslint)
// NOTE: Ensure not to go over 7.5 CPU/RAM due to EC2 node sizes and the jnlp container requirements
tinyPods.nodeBrowser([
resourceRequestCpu: '6',
tinyPods.node([
resourceRequestCpu: '2',
resourceRequestMemory: '4Gi',
resourceLimitCpu: '7.5',
resourceLimitMemory: '4Gi'
]) {
def props = readProperties(file: 'build.properties')

props = readProperties(file: 'build.properties')
String primaryBranch = props.primaryBranch
assert primaryBranch != null && primaryBranch != ""
def runAllTests = env.BRANCH_NAME == primaryBranch

stage("Merge") {
stage('Merge') {
// cancel build if primary branch doesn't merge cleanly
gitMerge(primaryBranch)
}

def platforms = [
[ os: "windows", browser: "chrome" ],
[ os: "windows", browser: "firefox" ],
[ os: "windows", browser: "MicrosoftEdge" ],
[ os: "macos", browser: "safari" ],
[ os: "macos", browser: "chrome" ],
[ os: "macos", browser: "firefox" ]
]

def cleanAndInstall = {
echo "Installing tools"
exec("git clean -fdx modules scratch js dist")
stage('Install') {
yarnInstall()
}

def processes = [:]

// Browser tests
for (int i = 0; i < platforms.size(); i++) {
def platform = platforms.get(i)

def buckets = platform.buckets ?: 1
for (int bucket = 1; bucket <= buckets; bucket++) {
def suffix = buckets == 1 ? "" : "-" + bucket

// closure variable - don't inline
def c_bucket = bucket

def name = "${platform.os}-${platform.browser}${suffix}"

processes[name] = {
stage(name) {
node("bedrock-${platform.os}") {
echo("Bedrock tests for ${name}")

echo("Checking out code on build node: $NODE_NAME")
checkout(scm)

// windows tends to not have username or email set
tinyGit.addAuthorConfig()
gitMerge(primaryBranch)

cleanAndInstall()
exec("yarn ci")

echo("Running browser tests")
runBrowserTests(name, platform.browser, platform.os, c_bucket, buckets, runAllTests)
}
}
}
}
stage("Validate changelog") {
// we use a changelog to run changie
exec("yarn changie-merge")
}

processes["headless-and-archive"] = {
stage("headless tests") {
runHeadlessTests(runAllTests)
}

if (env.BRANCH_NAME != primaryBranch) {
stage("Archive Build") {
exec("yarn tinymce-grunt prodBuild symlink:js")
archiveArtifacts artifacts: 'js/**', onlyIfSuccessful: true
}
stage('Type check') {
withEnv(["NODE_OPTIONS=--max-old-space-size=1936"]) {
exec("yarn ci-all-seq")
}
}

stage("Install tools") {
cleanAndInstall()
stage('Moxiedoc check') {
sh 'yarn tinymce-grunt shell:moxiedoc'
}
}

stage("Validate changelog") {
// we use a changelog to run changie
exec("yarn changie-merge")
def platforms = [
[ browser: 'chrome', provider: 'aws', buckets: 2 ],
// [ browser: 'edge', provider: 'aws', buckets: 2 ],
[ browser: 'firefox', provider: 'aws', buckets: 2 ],
[ browser: 'edge', provider: 'lambdatest', buckets: 1 ],
[ browser: 'safari', provider: 'lambdatest', buckets: 1 ],
[ browser: 'chrome', provider: 'lambdatest', os: 'macOS Sonoma', buckets: 1],
// [ browser: 'firefox', provider: 'lambdatest', os: 'macOS Sonoma', buckets: 1]
];

def processes = [:]
def runAllTests = env.BRANCH_NAME == props.primaryBranch

for (int i = 0; i < platforms.size(); i++) {
def platform = platforms.get(i)
def buckets = platform.buckets ?: 1
for (int bucket = 1; bucket <= buckets; bucket ++) {
def suffix = buckets == 1 ? "" : "-" + bucket
def s_bucket = "${bucket}"
def s_buckets = "${buckets}"
def name = "${platform.browser}-${platform.provider}${suffix}"
processes[name] = runTestPod(name, platform.browser, platform.provider, platform.os, s_bucket, s_buckets, runAllTests)
}
}

stage("Type check") {
withEnv(["NODE_OPTIONS=--max-old-space-size=1936"]) {
exec("yarn ci-all-seq")
processes['headless'] = {
tinyPods.nodeBrowser([
resourceRequestCpu: '2',
resourceRequestMemory: '4Gi',
resourceRequestEphemeralStorage: '16Gi',
resourceLimitCpu: '7.5',
resourceLimitMemory: '4Gi',
resourceLimitEphemeralStorage: '16Gi',
]) {
stage('Test-headless') {
yarnInstall()
withEnv(["NODE_OPTIONS=--max-old-space-size=1936"]) {
sh "yarn ci-all-seq"
}
}
}

stage("Moxiedoc check") {
exec("yarn tinymce-grunt shell:moxiedoc")
stage('test') {
grunt('list-changed-headless')
runHeadlessTests(runAllTests)
}
}
}

stage("Run Tests") {
grunt("list-changed-headless list-changed-browser")
// Run all the tests in parallel
stage('Run tests') {
echo "Running tests [runAll=${runAllTests}]"
parallel processes
}
}
}
}
3 changes: 2 additions & 1 deletion modules/tinymce/src/core/test/ts/browser/DragDropOverridesTest.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@ interface DataTransferSpec {
readonly files: File[];
}

describe('browser.tinymce.core.DragDropOverridesTest', () => {
// TODO TINY-10480: Investigate flaky tests
describe.skip('browser.tinymce.core.DragDropOverridesTest', () => {
context('Tests when the editor is inside the viewport', () => {
let events: DragEvent[] = [];
const hook = TinyHooks.bddSetup<Editor>({
Expand Down
3 changes: 2 additions & 1 deletion modules/tinymce/src/core/test/ts/browser/EditorAutoFocusTest.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@ import Editor from 'tinymce/core/api/Editor';
import EditorManager from 'tinymce/core/api/EditorManager';
import { RawEditorOptions } from 'tinymce/core/api/OptionTypes';

describe('browser.tinymce.core.EditorAutoFocusTest', () => {
// TODO TINY-10480: Investigate flaky tests
describe.skip('browser.tinymce.core.EditorAutoFocusTest', () => {
before(() => {
Insert.append(SugarBody.body(), SugarElement.fromHtml(`<div id="abc">
<div class="tinymce" id="mce_0">Editor_0</div>
Expand Down
3 changes: 2 additions & 1 deletion modules/tinymce/src/plugins/emoticons/test/ts/browser/ImageEmojiTest.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@ import Plugin from 'tinymce/plugins/emoticons/Plugin';

import { fakeEvent } from '../module/test/Utils';

describe('browser.tinymce.plugins.emoticons.ImageEmojiTest', () => {
// TODO TINY-10480: Investigate flaky tests
describe.skip('browser.tinymce.plugins.emoticons.ImageEmojiTest', () => {
const hook = TinyHooks.bddSetupLight<Editor>({
plugins: 'emoticons',
toolbar: 'emoticons',
Expand Down
3 changes: 2 additions & 1 deletion modules/tinymce/src/plugins/image/test/ts/browser/ImageResizeTest.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@ import Plugin from 'tinymce/plugins/image/Plugin';

import { assertCleanHtml, assertInputValue, generalTabSelectors, setInputValue } from '../module/Helpers';

describe('browser.tinymce.plugins.image.ImageResizeTest', () => {
// TODO TINY-10480: Investigate flaky tests
describe.skip('browser.tinymce.plugins.image.ImageResizeTest', () => {
const hook = TinyHooks.bddSetupLight<Editor>({
plugins: 'image',
toolbar: 'image',
Expand Down
3 changes: 2 additions & 1 deletion modules/tinymce/src/plugins/media/test/ts/browser/MediaEmbedTest.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@ import Plugin from 'tinymce/plugins/media/Plugin';

import * as Utils from '../module/test/Utils';

describe('browser.tinymce.plugins.media.core.MediaEmbedTest', () => {
// TODO TINY-10480: Investigate flaky tests
describe.skip('browser.tinymce.plugins.media.core.MediaEmbedTest', () => {
const hook = TinyHooks.bddSetupLight<Editor>({
plugins: [ 'media' ],
toolbar: 'media',
Expand Down
Loading

0 comments on commit b6fcc36

Please sign in to comment.