From 23af84c7d715316355c23757b4f0ee34f7f81776 Mon Sep 17 00:00:00 2001 From: Jens Deppe Date: Wed, 7 Nov 2018 15:26:04 -0800 Subject: [PATCH] GEODE-5995: Initial import of gradle docker plugin (#2790) Also includes: - Revert java version detection in original code (done for Java 9+ work). - Handle case where network interface disappears during docker teardown --- build.gradle | 1 - buildSrc/build.gradle | 15 + .../DefaultWorkerSemaphore.groovy | 71 ++ .../DockerizedJavaExecHandleBuilder.groovy | 100 +++ .../DockerizedTestExtension.groovy | 58 ++ .../DockerizedTestPlugin.groovy | 184 +++++ .../ExitCodeTolerantExecHandle.groovy | 92 +++ .../dockerizedtest/WorkerSemaphore.groovy | 28 + .../dockerizedtest/DockerizedExecHandle.java | 673 ++++++++++++++++++ .../DockerizedExecHandleRunner.java | 101 +++ .../ForciblyStoppableTestWorker.java | 45 ++ .../ForkingTestClassProcessor.java | 153 ++++ .../dockerizedtest/NoMemoryManager.java | 59 ++ .../plugins/dockerizedtest/TestExecuter.java | 116 +++ ...m.github.pedjak.dockerized-test.properties | 1 + 15 files changed, 1696 insertions(+), 1 deletion(-) create mode 100644 buildSrc/src/main/groovy/com/pedjak/gradle/plugins/dockerizedtest/DefaultWorkerSemaphore.groovy create mode 100644 buildSrc/src/main/groovy/com/pedjak/gradle/plugins/dockerizedtest/DockerizedJavaExecHandleBuilder.groovy create mode 100644 buildSrc/src/main/groovy/com/pedjak/gradle/plugins/dockerizedtest/DockerizedTestExtension.groovy create mode 100644 buildSrc/src/main/groovy/com/pedjak/gradle/plugins/dockerizedtest/DockerizedTestPlugin.groovy create mode 100644 buildSrc/src/main/groovy/com/pedjak/gradle/plugins/dockerizedtest/ExitCodeTolerantExecHandle.groovy create mode 100644 buildSrc/src/main/groovy/com/pedjak/gradle/plugins/dockerizedtest/WorkerSemaphore.groovy create mode 100755 buildSrc/src/main/java/com/pedjak/gradle/plugins/dockerizedtest/DockerizedExecHandle.java create mode 100644 buildSrc/src/main/java/com/pedjak/gradle/plugins/dockerizedtest/DockerizedExecHandleRunner.java create mode 100644 buildSrc/src/main/java/com/pedjak/gradle/plugins/dockerizedtest/ForciblyStoppableTestWorker.java create mode 100644 buildSrc/src/main/java/com/pedjak/gradle/plugins/dockerizedtest/ForkingTestClassProcessor.java create mode 100644 buildSrc/src/main/java/com/pedjak/gradle/plugins/dockerizedtest/NoMemoryManager.java create mode 100644 buildSrc/src/main/java/com/pedjak/gradle/plugins/dockerizedtest/TestExecuter.java create mode 100644 buildSrc/src/main/resources/META-INF/gradle-plugins/com.github.pedjak.dockerized-test.properties diff --git a/build.gradle b/build.gradle index 9b9a66150304..7db35d82365d 100755 --- a/build.gradle +++ b/build.gradle @@ -29,7 +29,6 @@ buildscript { classpath 'org.sonarsource.scanner.gradle:sonarqube-gradle-plugin:2.6.2' classpath "com.diffplug.spotless:spotless-plugin-gradle:3.10.0" classpath "me.champeau.gradle:jmh-gradle-plugin:0.4.7" - classpath "com.pedjak.gradle.plugins:dockerized-test:0.5.6.35-SNAPSHOT" classpath 'com.github.ben-manes:gradle-versions-plugin:0.17.0' classpath "com.netflix.nebula:nebula-project-plugin:4.0.1" } diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 00c9903256c7..819fc8a7187d 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -18,15 +18,30 @@ repositories { mavenCentral() + maven { url "http://geode-maven.s3-website-us-west-2.amazonaws.com" } } dependencies { compile (group: 'org.apache.geode', name: 'geode-junit', version: '1.3.0') { exclude group: 'org.apache.logging.log4j' } + compile gradleApi() + compile 'org.apache.commons:commons-lang3:3.3.2' + compile 'org.apache.maven:maven-artifact:3.3.3' + compile 'com.github.docker-java:docker-java:3.1.2-GEODE' compile group: 'junit', name: 'junit', version: '4.12' testAnnotationProcessor this.project +} +sourceSets { + main { + java { + srcDirs = [] + } + groovy { + srcDirs += ['src/main/java'] + } + } } diff --git a/buildSrc/src/main/groovy/com/pedjak/gradle/plugins/dockerizedtest/DefaultWorkerSemaphore.groovy b/buildSrc/src/main/groovy/com/pedjak/gradle/plugins/dockerizedtest/DefaultWorkerSemaphore.groovy new file mode 100644 index 000000000000..a42c4a1d74df --- /dev/null +++ b/buildSrc/src/main/groovy/com/pedjak/gradle/plugins/dockerizedtest/DefaultWorkerSemaphore.groovy @@ -0,0 +1,71 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.pedjak.gradle.plugins.dockerizedtest + +import org.gradle.api.Project +import org.gradle.api.tasks.testing.Test + +import java.util.concurrent.Semaphore + +class DefaultWorkerSemaphore implements WorkerSemaphore { + private int maxWorkers = Integer.MAX_VALUE + private Semaphore semaphore + private logger + + @Override + void acquire() { + semaphore().acquire() + logger.debug("Semaphore acquired, available: {}/{}", semaphore().availablePermits(), maxWorkers) + } + + @Override + void release() { + semaphore().release() + logger.debug("Semaphore released, available: {}/{}", semaphore().availablePermits(), maxWorkers) + } + + @Override + synchronized void applyTo(Project project) { + if (semaphore) return + if (!logger) { + logger = project.logger + } + + maxWorkers = project.tasks.withType(Test).findAll { + it.extensions.docker?.image != null + }.collect { + def v = it.maxParallelForks + it.maxParallelForks = 10000 + v + }.min() ?: 1 + semaphore() + } + + private synchronized setMaxWorkers(int num) { + if (this.@maxWorkers > num) { + this.@maxWorkers = num + } + } + + private synchronized Semaphore semaphore() { + if (semaphore == null) { + semaphore = new Semaphore(maxWorkers) + logger.lifecycle("Do not allow more than {} test workers", maxWorkers) + } + semaphore + } +} diff --git a/buildSrc/src/main/groovy/com/pedjak/gradle/plugins/dockerizedtest/DockerizedJavaExecHandleBuilder.groovy b/buildSrc/src/main/groovy/com/pedjak/gradle/plugins/dockerizedtest/DockerizedJavaExecHandleBuilder.groovy new file mode 100644 index 000000000000..70ac705ebb41 --- /dev/null +++ b/buildSrc/src/main/groovy/com/pedjak/gradle/plugins/dockerizedtest/DockerizedJavaExecHandleBuilder.groovy @@ -0,0 +1,100 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.pedjak.gradle.plugins.dockerizedtest + +import com.pedjak.gradle.plugins.dockerizedtest.DockerizedTestExtension +import com.pedjak.gradle.plugins.dockerizedtest.ExitCodeTolerantExecHandle +import com.pedjak.gradle.plugins.dockerizedtest.WorkerSemaphore +import org.gradle.api.internal.file.FileResolver +import org.gradle.initialization.BuildCancellationToken +import org.gradle.process.internal.* +import org.gradle.process.internal.streams.OutputStreamsForwarder + +import java.util.concurrent.Executor + +class DockerizedJavaExecHandleBuilder extends JavaExecHandleBuilder { + + def streamsHandler + def executor + def buildCancellationToken + private final DockerizedTestExtension extension + + private final WorkerSemaphore workersSemaphore + + DockerizedJavaExecHandleBuilder(DockerizedTestExtension extension, FileResolver fileResolver, Executor executor, BuildCancellationToken buildCancellationToken, WorkerSemaphore workersSemaphore) { + super(fileResolver, executor, buildCancellationToken) + this.extension = extension + this.executor = executor + this.buildCancellationToken = buildCancellationToken + this.workersSemaphore = workersSemaphore + } + + def StreamsHandler getStreamsHandler() { + StreamsHandler effectiveHandler; + if (this.streamsHandler != null) { + effectiveHandler = this.streamsHandler; + } else { + boolean shouldReadErrorStream = !redirectErrorStream; + effectiveHandler = new OutputStreamsForwarder(standardOutput, errorOutput, shouldReadErrorStream); + } + return effectiveHandler; + } + + ExecHandle build() { + + return new ExitCodeTolerantExecHandle(new DockerizedExecHandle(extension, getDisplayName(), + getWorkingDir(), + 'java', + allArguments, + getActualEnvironment(), + getStreamsHandler(), + inputHandler, + listeners, + redirectErrorStream, + timeoutMillis, + daemon, + executor, + buildCancellationToken), + workersSemaphore) + + } + + def timeoutMillis = Integer.MAX_VALUE + + @Override + AbstractExecHandleBuilder setTimeout(int timeoutMillis) { + this.timeoutMillis = timeoutMillis + return super.setTimeout(timeoutMillis) + } + + boolean redirectErrorStream + + @Override + AbstractExecHandleBuilder redirectErrorStream() { + redirectErrorStream = true + return super.redirectErrorStream() + } + + def listeners = [] + + @Override + AbstractExecHandleBuilder listener(ExecHandleListener listener) { + listeners << listener + return super.listener(listener) + } + +} diff --git a/buildSrc/src/main/groovy/com/pedjak/gradle/plugins/dockerizedtest/DockerizedTestExtension.groovy b/buildSrc/src/main/groovy/com/pedjak/gradle/plugins/dockerizedtest/DockerizedTestExtension.groovy new file mode 100644 index 000000000000..8a007f117fae --- /dev/null +++ b/buildSrc/src/main/groovy/com/pedjak/gradle/plugins/dockerizedtest/DockerizedTestExtension.groovy @@ -0,0 +1,58 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.pedjak.gradle.plugins.dockerizedtest + +import com.github.dockerjava.api.DockerClient + +class DockerizedTestExtension { + + String image + Map volumes + String user + + Closure beforeContainerCreate + + Closure afterContainerCreate + + Closure beforeContainerStart + + Closure afterContainerStart + + Closure afterContainerStop = { containerId, client -> + try { + client.removeContainerCmd(containerId).exec(); + } catch (Exception e) { + // ignore any error + } + } + + // could be a DockerClient instance or a closure that returns a DockerClient instance + private def clientOrClosure + + void setClient(clientOrClosure) { + this.clientOrClosure = clientOrClosure + } + + DockerClient getClient() { + if (clientOrClosure == null) return null + if (DockerClient.class.isAssignableFrom(clientOrClosure.getClass())) { + return (DockerClient) clientOrClosure; + } else { + return (DockerClient) ((Closure) clientOrClosure).call(); + } + } +} diff --git a/buildSrc/src/main/groovy/com/pedjak/gradle/plugins/dockerizedtest/DockerizedTestPlugin.groovy b/buildSrc/src/main/groovy/com/pedjak/gradle/plugins/dockerizedtest/DockerizedTestPlugin.groovy new file mode 100644 index 000000000000..84d5afc7a82a --- /dev/null +++ b/buildSrc/src/main/groovy/com/pedjak/gradle/plugins/dockerizedtest/DockerizedTestPlugin.groovy @@ -0,0 +1,184 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.pedjak.gradle.plugins.dockerizedtest + +import com.github.dockerjava.api.DockerClient +import com.github.dockerjava.core.DefaultDockerClientConfig +import com.github.dockerjava.core.DockerClientBuilder +import com.github.dockerjava.netty.NettyDockerCmdExecFactory +import org.apache.commons.lang3.SystemUtils +import org.apache.maven.artifact.versioning.ComparableVersion +import org.gradle.api.Action +import org.gradle.api.GradleException +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.tasks.testing.Test +import org.gradle.initialization.DefaultBuildCancellationToken +import org.gradle.internal.concurrent.DefaultExecutorFactory +import org.gradle.internal.concurrent.ExecutorFactory +import org.gradle.internal.operations.BuildOperationExecutor +import org.gradle.internal.remote.Address +import org.gradle.internal.remote.ConnectionAcceptor +import org.gradle.internal.remote.MessagingServer +import org.gradle.internal.remote.ObjectConnection +import org.gradle.internal.remote.internal.ConnectCompletion +import org.gradle.internal.remote.internal.IncomingConnector +import org.gradle.internal.remote.internal.hub.MessageHubBackedObjectConnection +import org.gradle.internal.remote.internal.inet.MultiChoiceAddress +import org.gradle.internal.time.Clock +import org.gradle.process.internal.JavaExecHandleFactory +import org.gradle.process.internal.worker.DefaultWorkerProcessFactory + +import javax.inject.Inject + +class DockerizedTestPlugin implements Plugin { + + def supportedVersion = '4.8' + def currentUser + def messagingServer + def static workerSemaphore = new DefaultWorkerSemaphore() + def memoryManager = new com.pedjak.gradle.plugins.dockerizedtest.NoMemoryManager() + + @Inject + DockerizedTestPlugin(MessagingServer messagingServer) { + this.currentUser = SystemUtils.IS_OS_WINDOWS ? "0" : "id -u".execute().text.trim() + this.messagingServer = new MessageServer(messagingServer.connector, messagingServer.executorFactory) + } + + void configureTest(project, test) { + def ext = test.extensions.create("docker", DockerizedTestExtension, [] as Object[]) + def startParameter = project.gradle.startParameter + ext.volumes = ["$startParameter.gradleUserHomeDir": "$startParameter.gradleUserHomeDir", + "$project.projectDir" : "$project.projectDir"] + ext.user = currentUser + test.doFirst { + def extension = test.extensions.docker + + if (extension?.image) { + + workerSemaphore.applyTo(test.project) + test.testExecuter = new com.pedjak.gradle.plugins.dockerizedtest.TestExecuter(newProcessBuilderFactory(project, extension, test.processBuilderFactory), actorFactory, moduleRegistry, services.get(BuildOperationExecutor), services.get(Clock)); + + if (!extension.client) { + extension.client = createDefaultClient() + } + } + + } + } + + DockerClient createDefaultClient() { + DockerClientBuilder.getInstance(DefaultDockerClientConfig.createDefaultConfigBuilder()) + .withDockerCmdExecFactory(new NettyDockerCmdExecFactory()) + .build() + } + + void apply(Project project) { + + boolean unsupportedVersion = new ComparableVersion(project.gradle.gradleVersion).compareTo(new ComparableVersion(supportedVersion)) < 0 + if (unsupportedVersion) throw new GradleException("dockerized-test plugin requires Gradle ${supportedVersion}+") + + project.tasks.withType(Test).each { test -> configureTest(project, test) } + project.tasks.whenTaskAdded { task -> + if (task instanceof Test) configureTest(project, task) + } + } + + def newProcessBuilderFactory(project, extension, defaultProcessBuilderFactory) { + + def executorFactory = new DefaultExecutorFactory() + def executor = executorFactory.create("Docker container link") + def buildCancellationToken = new DefaultBuildCancellationToken() + + def execHandleFactory = [newJavaExec: { -> + new com.pedjak.gradle.plugins.dockerizedtest.DockerizedJavaExecHandleBuilder(extension, project.fileResolver, executor, buildCancellationToken, workerSemaphore) + }] as JavaExecHandleFactory + new DefaultWorkerProcessFactory(defaultProcessBuilderFactory.loggingManager, + messagingServer, + defaultProcessBuilderFactory.workerImplementationFactory.classPathRegistry, + defaultProcessBuilderFactory.idGenerator, + defaultProcessBuilderFactory.gradleUserHomeDir, + defaultProcessBuilderFactory.workerImplementationFactory.temporaryFileProvider, + execHandleFactory, + defaultProcessBuilderFactory.workerImplementationFactory.jvmVersionDetector, + defaultProcessBuilderFactory.outputEventListener, + memoryManager + ) + } + + class MessageServer implements MessagingServer { + def IncomingConnector connector; + def ExecutorFactory executorFactory; + + public MessageServer(IncomingConnector connector, ExecutorFactory executorFactory) { + this.connector = connector; + this.executorFactory = executorFactory; + } + + public ConnectionAcceptor accept(Action action) { + return new ConnectionAcceptorDelegate(connector.accept(new ConnectEventAction(action, executorFactory), true)) + } + + + } + + class ConnectEventAction implements Action { + def action; + def executorFactory; + + public ConnectEventAction(Action action, executorFactory) { + this.executorFactory = executorFactory + this.action = action + } + + public void execute(ConnectCompletion completion) { + action.execute(new MessageHubBackedObjectConnection(executorFactory, completion)); + } + } + + class ConnectionAcceptorDelegate implements ConnectionAcceptor { + + MultiChoiceAddress address + + @Delegate + ConnectionAcceptor delegate + + ConnectionAcceptorDelegate(ConnectionAcceptor delegate) { + this.delegate = delegate + } + + Address getAddress() { + synchronized (delegate) + { + if (address == null) { + def remoteAddresses = NetworkInterface.networkInterfaces.findAll { + try { + return it.up && !it.loopback + } catch (SocketException ex) { + logger.warn("Unable to inspect interface " + it) + return false + } + }*.inetAddresses*.collect { it }.flatten() + def original = delegate.address + address = new MultiChoiceAddress(original.canonicalAddress, original.port, remoteAddresses) + } + } + address + } + } + +} \ No newline at end of file diff --git a/buildSrc/src/main/groovy/com/pedjak/gradle/plugins/dockerizedtest/ExitCodeTolerantExecHandle.groovy b/buildSrc/src/main/groovy/com/pedjak/gradle/plugins/dockerizedtest/ExitCodeTolerantExecHandle.groovy new file mode 100644 index 000000000000..ab1c6f119c2f --- /dev/null +++ b/buildSrc/src/main/groovy/com/pedjak/gradle/plugins/dockerizedtest/ExitCodeTolerantExecHandle.groovy @@ -0,0 +1,92 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.pedjak.gradle.plugins.dockerizedtest + +import com.pedjak.gradle.plugins.dockerizedtest.WorkerSemaphore +import org.gradle.process.ExecResult +import org.gradle.process.internal.ExecException +import org.gradle.process.internal.ExecHandle +import org.gradle.process.internal.ExecHandleListener + +/** + * All exit codes are normal + */ +class ExitCodeTolerantExecHandle implements ExecHandle { + + private final WorkerSemaphore testWorkerSemaphore + + @Delegate + private final ExecHandle delegate + + ExitCodeTolerantExecHandle(ExecHandle delegate, WorkerSemaphore testWorkerSemaphore) { + this.delegate = delegate + this.testWorkerSemaphore = testWorkerSemaphore + delegate.addListener(new ExecHandleListener() { + + @Override + void executionStarted(ExecHandle execHandle) { + // do nothing + } + + @Override + void executionFinished(ExecHandle execHandle, ExecResult execResult) { + testWorkerSemaphore.release() + } + }) + } + + ExecHandle start() { + testWorkerSemaphore.acquire() + try { + delegate.start() + } catch (Exception e) { + testWorkerSemaphore.release() + throw e + } + } + + private static class ExitCodeTolerantExecResult implements ExecResult { + + @Delegate + private final ExecResult delegate + + ExitCodeTolerantExecResult(ExecResult delegate) { + this.delegate = delegate + } + + ExecResult assertNormalExitValue() throws ExecException { + // no op because we are perfectly ok if the exit code is anything + // because Docker can complain about not being able to remove the used image + // although the tests completed fine + this + } + } + + private static class ExecHandleListenerFacade implements ExecHandleListener { + + @Delegate + private final ExecHandleListener delegate + + ExecHandleListenerFacade(ExecHandleListener delegate) { + this.delegate = delegate + } + + void executionFinished(ExecHandle execHandle, ExecResult execResult) { + delegate.executionFinished(execHandle, new ExitCodeTolerantExecResult(execResult)) + } + } +} diff --git a/buildSrc/src/main/groovy/com/pedjak/gradle/plugins/dockerizedtest/WorkerSemaphore.groovy b/buildSrc/src/main/groovy/com/pedjak/gradle/plugins/dockerizedtest/WorkerSemaphore.groovy new file mode 100644 index 000000000000..0268088a434a --- /dev/null +++ b/buildSrc/src/main/groovy/com/pedjak/gradle/plugins/dockerizedtest/WorkerSemaphore.groovy @@ -0,0 +1,28 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.pedjak.gradle.plugins.dockerizedtest + +import org.gradle.api.Project + +interface WorkerSemaphore { + + void acquire() + + void release() + + void applyTo(Project project) +} \ No newline at end of file diff --git a/buildSrc/src/main/java/com/pedjak/gradle/plugins/dockerizedtest/DockerizedExecHandle.java b/buildSrc/src/main/java/com/pedjak/gradle/plugins/dockerizedtest/DockerizedExecHandle.java new file mode 100755 index 000000000000..11e3ac6cedab --- /dev/null +++ b/buildSrc/src/main/java/com/pedjak/gradle/plugins/dockerizedtest/DockerizedExecHandle.java @@ -0,0 +1,673 @@ +/* + * Copyright 2010 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.pedjak.gradle.plugins.dockerizedtest; + +import static java.lang.String.format; + +import java.io.File; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.PipedInputStream; +import java.io.PipedOutputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +import javax.annotation.Nullable; + +import com.github.dockerjava.api.DockerClient; +import com.github.dockerjava.api.command.CreateContainerCmd; +import com.github.dockerjava.api.model.Frame; +import com.github.dockerjava.api.model.StreamType; +import com.github.dockerjava.api.model.WaitResponse; +import com.github.dockerjava.core.command.AttachContainerResultCallback; +import com.github.dockerjava.core.command.WaitContainerResultCallback; +import com.github.dockerjava.api.model.Bind; +import com.github.dockerjava.api.model.Volume; +import com.google.common.base.Joiner; +import groovy.lang.Closure; +import org.gradle.api.logging.Logger; +import org.gradle.api.logging.Logging; +import org.gradle.initialization.BuildCancellationToken; +import org.gradle.internal.UncheckedException; +import org.gradle.internal.event.ListenerBroadcast; +import org.gradle.internal.operations.CurrentBuildOperationPreservingRunnable; +import org.gradle.process.ExecResult; +import org.gradle.process.internal.ExecException; +import org.gradle.process.internal.ExecHandle; +import org.gradle.process.internal.ExecHandleListener; +import org.gradle.process.internal.ExecHandleShutdownHookAction; +import org.gradle.process.internal.ExecHandleState; +import org.gradle.process.internal.ProcessSettings; +import org.gradle.process.internal.StreamsHandler; +import org.gradle.process.internal.shutdown.ShutdownHookActionRegister; + +/** + * Default implementation for the ExecHandle interface. + * + *

State flows

+ * + *
    + *
  • INIT -> STARTED -> [SUCCEEDED|FAILED|ABORTED|DETACHED]
    • + *
  • INIT -> FAILED
    • + *
  • INIT -> STARTED -> DETACHED -> ABORTED
    • + *
+ * + * State is controlled on all control methods: + *
    + *
  • {@link #start()} allowed when state is INIT
    • + *
  • {@link #abort()} allowed when state is STARTED or DETACHED
    • + *
+ */ +public class DockerizedExecHandle implements ExecHandle, ProcessSettings { + + private static final Logger LOGGER = Logging.getLogger(DockerizedExecHandle.class); + + private final String displayName; + + /** + * The working directory of the process. + */ + private final File directory; + + /** + * The executable to run. + */ + private final String command; + + /** + * Arguments to pass to the executable. + */ + private final List arguments; + + /** + * The variables to set in the environment the executable is run in. + */ + private final Map environment; + private final StreamsHandler outputHandler; + private final StreamsHandler inputHandler; + private final boolean redirectErrorStream; + private int timeoutMillis; + private boolean daemon; + + /** + * Lock to guard all mutable state + */ + private final Lock lock; + private final Condition stateChanged; + + private final Executor executor; + + /** + * State of this ExecHandle. + */ + private ExecHandleState state; + + /** + * When not null, the runnable that is waiting + */ + private DockerizedExecHandleRunner execHandleRunner; + + private ExecResultImpl execResult; + + private final ListenerBroadcast broadcast; + + private final ExecHandleShutdownHookAction shutdownHookAction; + + private final BuildCancellationToken buildCancellationToken; + + private final DockerizedTestExtension testExtension; + + public DockerizedExecHandle(DockerizedTestExtension testExtension, String displayName, + File directory, String command, List arguments, + Map environment, StreamsHandler outputHandler, + StreamsHandler inputHandler, + List listeners, boolean redirectErrorStream, + int timeoutMillis, boolean daemon, + Executor executor, BuildCancellationToken buildCancellationToken) { + this.displayName = displayName; + this.directory = directory; + this.command = command; + this.arguments = arguments; + this.environment = environment; + this.outputHandler = outputHandler; + this.inputHandler = inputHandler; + this.redirectErrorStream = redirectErrorStream; + this.timeoutMillis = timeoutMillis; + this.daemon = daemon; + this.executor = executor; + this.lock = new ReentrantLock(); + this.stateChanged = lock.newCondition(); + this.state = ExecHandleState.INIT; + this.buildCancellationToken = buildCancellationToken; + this.testExtension = testExtension; + shutdownHookAction = new ExecHandleShutdownHookAction(this); + broadcast = new ListenerBroadcast(ExecHandleListener.class); + broadcast.addAll(listeners); + } + + public File getDirectory() { + return directory; + } + + public String getCommand() { + return command; + } + + public boolean isDaemon() { + return daemon; + } + + @Override + public String toString() { + return displayName; + } + + public List getArguments() { + return Collections.unmodifiableList(arguments); + } + + public Map getEnvironment() { + return Collections.unmodifiableMap(environment); + } + + public ExecHandleState getState() { + lock.lock(); + try { + return state; + } finally { + lock.unlock(); + } + } + + private void setState(ExecHandleState state) { + lock.lock(); + try { + LOGGER.debug("Changing state to: {}", state); + this.state = state; + this.stateChanged.signalAll(); + } finally { + lock.unlock(); + } + } + + private boolean stateIn(ExecHandleState... states) { + lock.lock(); + try { + return Arrays.asList(states).contains(this.state); + } finally { + lock.unlock(); + } + } + + private void setEndStateInfo(ExecHandleState newState, int exitValue, Throwable failureCause) { + ShutdownHookActionRegister.removeAction(shutdownHookAction); + buildCancellationToken.removeCallback(shutdownHookAction); + ExecHandleState currentState; + lock.lock(); + try { + currentState = this.state; + } finally { + lock.unlock(); + } + + ExecResultImpl + newResult = + new ExecResultImpl(exitValue, execExceptionFor(failureCause, currentState), displayName); + if (!currentState.isTerminal() && newState != ExecHandleState.DETACHED) { + try { + broadcast.getSource().executionFinished(this, newResult); + } catch (Exception e) { + newResult = new ExecResultImpl(exitValue, execExceptionFor(e, currentState), displayName); + } + } + + lock.lock(); + try { + setState(newState); + this.execResult = newResult; + } finally { + lock.unlock(); + } + + LOGGER.debug("Process '{}' finished with exit value {} (state: {})", displayName, exitValue, + newState); + } + + @Nullable + private ExecException execExceptionFor(Throwable failureCause, ExecHandleState currentState) { + return failureCause != null + ? new ExecException(failureMessageFor(currentState), failureCause) + : null; + } + + private String failureMessageFor(ExecHandleState currentState) { + return currentState == ExecHandleState.STARTING + ? format("A problem occurred starting process '%s'", displayName) + : format("A problem occurred waiting for process '%s' to complete.", displayName); + } + + public ExecHandle start() { + LOGGER.info("Starting process '{}'. Working directory: {} Command: {}", + displayName, directory, command + ' ' + Joiner.on(' ').useForNull("null").join(arguments)); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Environment for process '{}': {}", displayName, environment); + } + lock.lock(); + try { + if (!stateIn(ExecHandleState.INIT)) { + throw new IllegalStateException( + format("Cannot start process '%s' because it has already been started", displayName)); + } + setState(ExecHandleState.STARTING); + + execHandleRunner = + new DockerizedExecHandleRunner(this, new CompositeStreamsHandler(), executor); + executor.execute(new CurrentBuildOperationPreservingRunnable(execHandleRunner)); + + while (stateIn(ExecHandleState.STARTING)) { + LOGGER.debug("Waiting until process started: {}.", displayName); + try { + if (!stateChanged.await(30, TimeUnit.SECONDS)) { + execHandleRunner.abortProcess(); + throw new RuntimeException("Giving up on " + execHandleRunner); + } + } catch (InterruptedException e) { + //ok, wrapping up + } + } + + if (execResult != null) { + execResult.rethrowFailure(); + } + + LOGGER.info("Successfully started process '{}'", displayName); + } finally { + lock.unlock(); + } + return this; + } + + public void abort() { + lock.lock(); + try { + if (stateIn(ExecHandleState.SUCCEEDED, ExecHandleState.FAILED, ExecHandleState.ABORTED)) { + return; + } + if (!stateIn(ExecHandleState.STARTED, ExecHandleState.DETACHED)) { + throw new IllegalStateException( + format("Cannot abort process '%s' because it is not in started or detached state", + displayName)); + } + this.execHandleRunner.abortProcess(); + this.waitForFinish(); + } finally { + lock.unlock(); + } + } + + public ExecResult waitForFinish() { + lock.lock(); + try { + while (!state.isTerminal()) { + try { + stateChanged.await(); + } catch (InterruptedException e) { + //ok, wrapping up... + throw UncheckedException.throwAsUncheckedException(e); + } + } + } finally { + lock.unlock(); + } + + // At this point: + // If in daemon mode, the process has started successfully and all streams to the process have been closed + // If in fork mode, the process has completed and all cleanup has been done + // In both cases, all asynchronous work for the process has completed and we're done + + return result(); + } + + private ExecResult result() { + lock.lock(); + try { + return execResult.rethrowFailure(); + } finally { + lock.unlock(); + } + } + + void detached() { + setEndStateInfo(ExecHandleState.DETACHED, 0, null); + } + + void started() { + ShutdownHookActionRegister.addAction(shutdownHookAction); + buildCancellationToken.addCallback(shutdownHookAction); + setState(ExecHandleState.STARTED); + broadcast.getSource().executionStarted(this); + } + + void finished(int exitCode) { + if (exitCode != 0) { + setEndStateInfo(ExecHandleState.FAILED, exitCode, null); + } else { + setEndStateInfo(ExecHandleState.SUCCEEDED, 0, null); + } + } + + void aborted(int exitCode) { + if (exitCode == 0) { + // This can happen on Windows + exitCode = -1; + } + setEndStateInfo(ExecHandleState.ABORTED, exitCode, null); + } + + void failed(Throwable failureCause) { + setEndStateInfo(ExecHandleState.FAILED, -1, failureCause); + } + + public void addListener(ExecHandleListener listener) { + broadcast.add(listener); + } + + public void removeListener(ExecHandleListener listener) { + broadcast.remove(listener); + } + + public String getDisplayName() { + return displayName; + } + + public boolean getRedirectErrorStream() { + return redirectErrorStream; + } + + public int getTimeout() { + return timeoutMillis; + } + + public Process runContainer() { + try { + DockerClient client = testExtension.getClient(); + CreateContainerCmd createCmd = client.createContainerCmd(testExtension.getImage().toString()) + .withTty(false) + .withStdinOpen(true) + .withWorkingDir(directory.getAbsolutePath()); + + createCmd.withEnv(getEnv()); + + String user = testExtension.getUser(); + if (user != null) { + createCmd.withUser(user); + } + bindVolumes(createCmd); + List cmdLine = new ArrayList(); + cmdLine.add(command); + cmdLine.addAll(arguments); + createCmd.withCmd(cmdLine); + + invokeIfNotNull(testExtension.getBeforeContainerCreate(), createCmd, client); + String containerId = createCmd.exec().getId(); + invokeIfNotNull(testExtension.getAfterContainerCreate(), containerId, client); + + invokeIfNotNull(testExtension.getBeforeContainerStart(), containerId, client); + client.startContainerCmd(containerId).exec(); + invokeIfNotNull(testExtension.getAfterContainerStart(), containerId, client); + + if (!client.inspectContainerCmd(containerId).exec().getState().getRunning()) { + throw new RuntimeException("Container " + containerId + " not running!"); + } + + Process + proc = + new DockerizedProcess(client, containerId, testExtension.getAfterContainerStop()); + + return proc; + } catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + } + + private void invokeIfNotNull(Closure closure, Object... args) { + if (closure != null) { + int l = closure.getParameterTypes().length; + Object[] nargs; + if (l < args.length) { + nargs = new Object[l]; + System.arraycopy(args, 0, nargs, 0, l); + } else { + nargs = args; + } + closure.call(nargs); + } + } + + private List getEnv() { + List env = new ArrayList(); + for (Map.Entry e : environment.entrySet()) { + env.add(e.getKey() + "=" + e.getValue()); + } + return env; + } + + private void bindVolumes(CreateContainerCmd cmd) { + List volumes = new ArrayList(); + List binds = new ArrayList(); + for (Iterator it = testExtension.getVolumes().entrySet().iterator(); it.hasNext(); ) { + Map.Entry e = (Map.Entry) it.next(); + Volume volume = new Volume(e.getValue().toString()); + Bind bind = new Bind(e.getKey().toString(), volume); + binds.add(bind); + volumes.add(volume); + } + cmd.withVolumes(volumes).withBinds(binds); + } + + private static class ExecResultImpl implements ExecResult { + private final int exitValue; + private final ExecException failure; + private final String displayName; + + ExecResultImpl(int exitValue, ExecException failure, String displayName) { + this.exitValue = exitValue; + this.failure = failure; + this.displayName = displayName; + } + + public int getExitValue() { + return exitValue; + } + + public ExecResult assertNormalExitValue() throws ExecException { + // all exit values are ok +// if (exitValue != 0) { +// throw new ExecException(format("Process '%s' finished with non-zero exit value %d", displayName, exitValue)); +// } + return this; + } + + public ExecResult rethrowFailure() throws ExecException { + if (failure != null) { + throw failure; + } + return this; + } + + @Override + public String toString() { + return "{exitValue=" + exitValue + ", failure=" + failure + "}"; + } + } + + private class CompositeStreamsHandler implements StreamsHandler { + @Override + public void connectStreams(Process process, String processName, Executor executor) { + inputHandler.connectStreams(process, processName, executor); + outputHandler.connectStreams(process, processName, executor); + } + + @Override + public void start() { + inputHandler.start(); + outputHandler.start(); + } + + @Override + public void stop() { + inputHandler.stop(); + outputHandler.stop(); + } + } + + private class DockerizedProcess extends Process { + + private final DockerClient dockerClient; + private final String containerId; + private final Closure afterContainerStop; + + private final PipedOutputStream stdInWriteStream = new PipedOutputStream(); + private final PipedInputStream stdOutReadStream = new PipedInputStream(); + private final PipedInputStream stdErrReadStream = new PipedInputStream(); + private final PipedInputStream stdInReadStream = new PipedInputStream(stdInWriteStream); + private final PipedOutputStream stdOutWriteStream = new PipedOutputStream(stdOutReadStream); + private final PipedOutputStream stdErrWriteStream = new PipedOutputStream(stdErrReadStream); + + private final CountDownLatch finished = new CountDownLatch(1); + private AtomicInteger exitCode = new AtomicInteger(); + private final AttachContainerResultCallback + attachContainerResultCallback = + new AttachContainerResultCallback() { + @Override + public void onNext(Frame frame) { + try { + if (frame.getStreamType().equals(StreamType.STDOUT)) { + stdOutWriteStream.write(frame.getPayload()); + } else if (frame.getStreamType().equals(StreamType.STDERR)) { + stdErrWriteStream.write(frame.getPayload()); + } + } catch (Exception e) { + LOGGER.error("Error while writing to stream:", e); + } + super.onNext(frame); + } + }; + + private final WaitContainerResultCallback + waitContainerResultCallback = + new WaitContainerResultCallback() { + @Override + public void onNext(WaitResponse waitResponse) { + exitCode.set(waitResponse.getStatusCode()); + try { + attachContainerResultCallback.close(); + attachContainerResultCallback.awaitCompletion(); + stdOutWriteStream.close(); + stdErrWriteStream.close(); + } catch (Exception e) { + LOGGER.debug("Error by detaching streams", e); + } finally { + try { + invokeIfNotNull(afterContainerStop, containerId, dockerClient); + } catch (Exception e) { + LOGGER.debug("Exception thrown at invoking afterContainerStop", e); + } finally { + finished.countDown(); + } + + } + + + } + }; + + public DockerizedProcess(final DockerClient dockerClient, final String containerId, + final Closure afterContainerStop) throws Exception { + this.dockerClient = dockerClient; + this.containerId = containerId; + this.afterContainerStop = afterContainerStop; + attachStreams(); + dockerClient.waitContainerCmd(containerId).exec(waitContainerResultCallback); + } + + private void attachStreams() throws Exception { + dockerClient.attachContainerCmd(containerId) + .withFollowStream(true) + .withStdOut(true) + .withStdErr(true) + .withStdIn(stdInReadStream) + .exec(attachContainerResultCallback); + if (!attachContainerResultCallback.awaitStarted(10, TimeUnit.SECONDS)) { + LOGGER.warn("Not attached to container " + containerId + " within 10secs"); + throw new RuntimeException("Not attached to container " + containerId + " within 10secs"); + } + } + + @Override + public OutputStream getOutputStream() { + return stdInWriteStream; + } + + @Override + public InputStream getInputStream() { + return stdOutReadStream; + } + + @Override + public InputStream getErrorStream() { + return stdErrReadStream; + } + + @Override + public int waitFor() throws InterruptedException { + finished.await(); + return exitCode.get(); + } + + @Override + public int exitValue() { + if (finished.getCount() > 0) { + throw new IllegalThreadStateException("docker process still running"); + } + return exitCode.get(); + } + + @Override + public void destroy() { + dockerClient.killContainerCmd(containerId).exec(); + } + + @Override + public String toString() { + return "Container " + containerId + " on " + dockerClient.toString(); + } + } + +} diff --git a/buildSrc/src/main/java/com/pedjak/gradle/plugins/dockerizedtest/DockerizedExecHandleRunner.java b/buildSrc/src/main/java/com/pedjak/gradle/plugins/dockerizedtest/DockerizedExecHandleRunner.java new file mode 100644 index 000000000000..c5678303460f --- /dev/null +++ b/buildSrc/src/main/java/com/pedjak/gradle/plugins/dockerizedtest/DockerizedExecHandleRunner.java @@ -0,0 +1,101 @@ +/* + * Copyright 2010 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.pedjak.gradle.plugins.dockerizedtest; + +import java.util.concurrent.Executor; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +import org.gradle.api.logging.Logger; +import org.gradle.api.logging.Logging; +import org.gradle.process.internal.StreamsHandler; + +public class DockerizedExecHandleRunner implements Runnable { + private static final Logger + LOGGER = + Logging.getLogger(org.gradle.process.internal.ExecHandleRunner.class); + + private final DockerizedExecHandle execHandle; + private final Lock lock = new ReentrantLock(); + private final Executor executor; + + private Process process; + private boolean aborted; + private final StreamsHandler streamsHandler; + + public DockerizedExecHandleRunner(DockerizedExecHandle execHandle, StreamsHandler streamsHandler, + Executor executor) { + this.executor = executor; + if (execHandle == null) { + throw new IllegalArgumentException("execHandle == null!"); + } + this.streamsHandler = streamsHandler; + this.execHandle = execHandle; + } + + public void abortProcess() { + lock.lock(); + try { + aborted = true; + if (process != null) { + LOGGER.debug("Abort requested. Destroying process: {}.", execHandle.getDisplayName()); + process.destroy(); + } + } finally { + lock.unlock(); + } + } + + public void run() { + try { + process = execHandle.runContainer(); + streamsHandler.connectStreams(process, execHandle.getDisplayName(), executor); + + execHandle.started(); + + LOGGER.debug("waiting until streams are handled..."); + streamsHandler.start(); + if (execHandle.isDaemon()) { + streamsHandler.stop(); + detached(); + } else { + int exitValue = process.waitFor(); + streamsHandler.stop(); + completed(exitValue); + } + } catch (Throwable t) { + execHandle.failed(t); + } + } + + private void completed(int exitValue) { + if (aborted) { + execHandle.aborted(exitValue); + } else { + execHandle.finished(exitValue); + } + } + + private void detached() { + execHandle.detached(); + } + + public String toString() { + return "Handler for " + process.toString(); + } +} + diff --git a/buildSrc/src/main/java/com/pedjak/gradle/plugins/dockerizedtest/ForciblyStoppableTestWorker.java b/buildSrc/src/main/java/com/pedjak/gradle/plugins/dockerizedtest/ForciblyStoppableTestWorker.java new file mode 100644 index 000000000000..fbbe48e50264 --- /dev/null +++ b/buildSrc/src/main/java/com/pedjak/gradle/plugins/dockerizedtest/ForciblyStoppableTestWorker.java @@ -0,0 +1,45 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.pedjak.gradle.plugins.dockerizedtest; + +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.TimeUnit; + +import org.gradle.api.internal.tasks.testing.WorkerTestClassProcessorFactory; +import org.gradle.api.internal.tasks.testing.worker.TestWorker; + +public class ForciblyStoppableTestWorker extends TestWorker { + private static final int SHUTDOWN_TIMEOUT = 60; // secs + + public ForciblyStoppableTestWorker(WorkerTestClassProcessorFactory factory) { + super(factory); + } + + @Override + public void stop() { + new Timer(true).schedule(new TimerTask() { + @Override + public void run() { + System.err.println("Worker process did not shutdown gracefully within " + SHUTDOWN_TIMEOUT + + "s, forcing it now"); + Runtime.getRuntime().halt(-100); + } + }, TimeUnit.SECONDS.toMillis(SHUTDOWN_TIMEOUT)); + super.stop(); + } +} diff --git a/buildSrc/src/main/java/com/pedjak/gradle/plugins/dockerizedtest/ForkingTestClassProcessor.java b/buildSrc/src/main/java/com/pedjak/gradle/plugins/dockerizedtest/ForkingTestClassProcessor.java new file mode 100644 index 000000000000..d6f409b461ab --- /dev/null +++ b/buildSrc/src/main/java/com/pedjak/gradle/plugins/dockerizedtest/ForkingTestClassProcessor.java @@ -0,0 +1,153 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.pedjak.gradle.plugins.dockerizedtest; + +import java.io.File; +import java.net.URL; +import java.util.List; + +import org.gradle.api.Action; +import org.gradle.api.internal.classpath.ModuleRegistry; +import org.gradle.api.internal.tasks.testing.TestClassProcessor; +import org.gradle.api.internal.tasks.testing.TestClassRunInfo; +import org.gradle.api.internal.tasks.testing.TestResultProcessor; +import org.gradle.api.internal.tasks.testing.WorkerTestClassProcessorFactory; +import org.gradle.api.internal.tasks.testing.worker.RemoteTestClassProcessor; +import org.gradle.api.internal.tasks.testing.worker.TestEventSerializer; +import org.gradle.internal.remote.ObjectConnection; +import org.gradle.process.JavaForkOptions; +import org.gradle.process.internal.worker.WorkerProcess; +import org.gradle.process.internal.worker.WorkerProcessBuilder; +import org.gradle.process.internal.worker.WorkerProcessFactory; +import org.gradle.util.CollectionUtils; + +public class ForkingTestClassProcessor implements TestClassProcessor { + private final WorkerProcessFactory workerFactory; + private final WorkerTestClassProcessorFactory processorFactory; + private final JavaForkOptions options; + private final Iterable classPath; + private final Action buildConfigAction; + private final ModuleRegistry moduleRegistry; + private RemoteTestClassProcessor remoteProcessor; + private WorkerProcess workerProcess; + private TestResultProcessor resultProcessor; + + public ForkingTestClassProcessor(WorkerProcessFactory workerFactory, + WorkerTestClassProcessorFactory processorFactory, + JavaForkOptions options, Iterable classPath, + Action buildConfigAction, + ModuleRegistry moduleRegistry) { + this.workerFactory = workerFactory; + this.processorFactory = processorFactory; + this.options = options; + this.classPath = classPath; + this.buildConfigAction = buildConfigAction; + this.moduleRegistry = moduleRegistry; + } + + @Override + public void startProcessing(TestResultProcessor resultProcessor) { + this.resultProcessor = resultProcessor; + } + + @Override + public void processTestClass(TestClassRunInfo testClass) { + int i = 0; + RuntimeException exception = null; + while (remoteProcessor == null && i < 10) { + try { + remoteProcessor = forkProcess(); + exception = null; + break; + } catch (RuntimeException e) { + exception = e; + i++; + } + } + + if (exception != null) { + throw exception; + } + remoteProcessor.processTestClass(testClass); + } + + RemoteTestClassProcessor forkProcess() { + WorkerProcessBuilder + builder = + workerFactory.create(new ForciblyStoppableTestWorker(processorFactory)); + builder.setBaseName("Gradle Test Executor"); + builder.setImplementationClasspath(getTestWorkerImplementationClasspath()); + builder.applicationClasspath(classPath); + options.copyTo(builder.getJavaCommand()); + buildConfigAction.execute(builder); + + workerProcess = builder.build(); + workerProcess.start(); + + ObjectConnection connection = workerProcess.getConnection(); + connection.useParameterSerializers(TestEventSerializer.create()); + connection.addIncoming(TestResultProcessor.class, resultProcessor); + RemoteTestClassProcessor + remoteProcessor = + connection.addOutgoing(RemoteTestClassProcessor.class); + connection.connect(); + remoteProcessor.startProcessing(); + return remoteProcessor; + } + + List getTestWorkerImplementationClasspath() { + return CollectionUtils.flattenCollections(URL.class, + moduleRegistry.getModule("gradle-core-api").getImplementationClasspath().getAsURLs(), + moduleRegistry.getModule("gradle-core").getImplementationClasspath().getAsURLs(), + moduleRegistry.getModule("gradle-logging").getImplementationClasspath().getAsURLs(), + moduleRegistry.getModule("gradle-messaging").getImplementationClasspath().getAsURLs(), + moduleRegistry.getModule("gradle-base-services").getImplementationClasspath().getAsURLs(), + moduleRegistry.getModule("gradle-cli").getImplementationClasspath().getAsURLs(), + moduleRegistry.getModule("gradle-native").getImplementationClasspath().getAsURLs(), + moduleRegistry.getModule("gradle-testing-base").getImplementationClasspath().getAsURLs(), + moduleRegistry.getModule("gradle-testing-jvm").getImplementationClasspath().getAsURLs(), + moduleRegistry.getModule("gradle-process-services").getImplementationClasspath() + .getAsURLs(), + moduleRegistry.getExternalModule("slf4j-api").getImplementationClasspath().getAsURLs(), + moduleRegistry.getExternalModule("jul-to-slf4j").getImplementationClasspath().getAsURLs(), + moduleRegistry.getExternalModule("native-platform").getImplementationClasspath() + .getAsURLs(), + moduleRegistry.getExternalModule("kryo").getImplementationClasspath().getAsURLs(), + moduleRegistry.getExternalModule("commons-lang").getImplementationClasspath().getAsURLs(), + moduleRegistry.getExternalModule("junit").getImplementationClasspath().getAsURLs(), + ForkingTestClassProcessor.class.getProtectionDomain().getCodeSource().getLocation() + ); + } + + @Override + public void stop() { + if (remoteProcessor != null) { + try { + remoteProcessor.stop(); + workerProcess.waitForStop(); + } finally { + // do nothing + } + } + } + + @Override + public void stopNow() { + stop(); // TODO need anything else ?? + } + +} diff --git a/buildSrc/src/main/java/com/pedjak/gradle/plugins/dockerizedtest/NoMemoryManager.java b/buildSrc/src/main/java/com/pedjak/gradle/plugins/dockerizedtest/NoMemoryManager.java new file mode 100644 index 000000000000..3a8be604f006 --- /dev/null +++ b/buildSrc/src/main/java/com/pedjak/gradle/plugins/dockerizedtest/NoMemoryManager.java @@ -0,0 +1,59 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.pedjak.gradle.plugins.dockerizedtest; + +import org.gradle.process.internal.health.memory.JvmMemoryStatusListener; +import org.gradle.process.internal.health.memory.MemoryHolder; +import org.gradle.process.internal.health.memory.MemoryManager; +import org.gradle.process.internal.health.memory.OsMemoryStatusListener; + +public class NoMemoryManager implements MemoryManager { + @Override + public void addListener(JvmMemoryStatusListener jvmMemoryStatusListener) { + + } + + @Override + public void addListener(OsMemoryStatusListener osMemoryStatusListener) { + + } + + @Override + public void removeListener(JvmMemoryStatusListener jvmMemoryStatusListener) { + + } + + @Override + public void removeListener(OsMemoryStatusListener osMemoryStatusListener) { + + } + + @Override + public void addMemoryHolder(MemoryHolder memoryHolder) { + + } + + @Override + public void removeMemoryHolder(MemoryHolder memoryHolder) { + + } + + @Override + public void requestFreeMemory(long l) { + + } +} diff --git a/buildSrc/src/main/java/com/pedjak/gradle/plugins/dockerizedtest/TestExecuter.java b/buildSrc/src/main/java/com/pedjak/gradle/plugins/dockerizedtest/TestExecuter.java new file mode 100644 index 000000000000..21a07ce61398 --- /dev/null +++ b/buildSrc/src/main/java/com/pedjak/gradle/plugins/dockerizedtest/TestExecuter.java @@ -0,0 +1,116 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.pedjak.gradle.plugins.dockerizedtest; + +import java.io.File; +import java.util.Set; +import java.util.UUID; + +import com.google.common.collect.ImmutableSet; +import org.gradle.api.file.FileTree; +import org.gradle.api.internal.classpath.ModuleRegistry; +import org.gradle.api.internal.tasks.testing.JvmTestExecutionSpec; +import org.gradle.api.internal.tasks.testing.TestClassProcessor; +import org.gradle.api.internal.tasks.testing.TestFramework; +import org.gradle.api.internal.tasks.testing.TestResultProcessor; +import org.gradle.api.internal.tasks.testing.WorkerTestClassProcessorFactory; +import org.gradle.api.internal.tasks.testing.detection.DefaultTestClassScanner; +import org.gradle.api.internal.tasks.testing.detection.TestFrameworkDetector; +import org.gradle.api.internal.tasks.testing.processors.MaxNParallelTestClassProcessor; +import org.gradle.api.internal.tasks.testing.processors.RestartEveryNTestClassProcessor; +import org.gradle.api.internal.tasks.testing.processors.TestMainAction; +import org.gradle.internal.Factory; +import org.gradle.internal.actor.ActorFactory; +import org.gradle.internal.operations.BuildOperationExecutor; +import org.gradle.internal.time.Clock; +import org.gradle.process.internal.worker.WorkerProcessFactory; + +public class TestExecuter + implements org.gradle.api.internal.tasks.testing.TestExecuter { + private final WorkerProcessFactory workerFactory; + private final ActorFactory actorFactory; + private final ModuleRegistry moduleRegistry; + private final BuildOperationExecutor buildOperationExecutor; + private final Clock clock; + private TestClassProcessor processor; + + public TestExecuter(WorkerProcessFactory workerFactory, ActorFactory actorFactory, + ModuleRegistry moduleRegistry, BuildOperationExecutor buildOperationExecutor, + Clock clock) { + this.workerFactory = workerFactory; + this.actorFactory = actorFactory; + this.moduleRegistry = moduleRegistry; + this.buildOperationExecutor = buildOperationExecutor; + this.clock = clock; + } + + @Override + public void execute(final JvmTestExecutionSpec testExecutionSpec, + TestResultProcessor testResultProcessor) { + final TestFramework testFramework = testExecutionSpec.getTestFramework(); + final WorkerTestClassProcessorFactory testInstanceFactory = testFramework.getProcessorFactory(); + final Set classpath = ImmutableSet.copyOf(testExecutionSpec.getClasspath()); + final Factory forkingProcessorFactory = new Factory() { + public TestClassProcessor create() { + return new ForkingTestClassProcessor(workerFactory, testInstanceFactory, + testExecutionSpec.getJavaForkOptions(), + classpath, testFramework.getWorkerConfigurationAction(), moduleRegistry); + } + }; + Factory reforkingProcessorFactory = new Factory() { + public TestClassProcessor create() { + return new RestartEveryNTestClassProcessor(forkingProcessorFactory, + testExecutionSpec.getForkEvery()); + } + }; + + processor = new MaxNParallelTestClassProcessor(testExecutionSpec.getMaxParallelForks(), + reforkingProcessorFactory, actorFactory); + + final FileTree testClassFiles = testExecutionSpec.getCandidateClassFiles(); + + Runnable detector; + if (testExecutionSpec.isScanForTestClasses()) { + TestFrameworkDetector + testFrameworkDetector = + testExecutionSpec.getTestFramework().getDetector(); + testFrameworkDetector.setTestClasses(testExecutionSpec.getTestClassesDirs().getFiles()); + testFrameworkDetector.setTestClasspath(classpath); + detector = new DefaultTestClassScanner(testClassFiles, testFrameworkDetector, processor); + } else { + detector = new DefaultTestClassScanner(testClassFiles, null, processor); + } + + Object testTaskOperationId; + + try { + testTaskOperationId = buildOperationExecutor.getCurrentOperation().getParentId(); + } catch (Exception e) { + testTaskOperationId = UUID.randomUUID(); + } + + new TestMainAction(detector, processor, testResultProcessor, clock, testTaskOperationId, + testExecutionSpec.getPath(), "Gradle Test Run " + testExecutionSpec.getIdentityPath()) + .run(); + } + + public void stopNow() { + if (processor != null) { + processor.stopNow(); + } + } +} diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/com.github.pedjak.dockerized-test.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/com.github.pedjak.dockerized-test.properties new file mode 100644 index 000000000000..1cfe2cb87aab --- /dev/null +++ b/buildSrc/src/main/resources/META-INF/gradle-plugins/com.github.pedjak.dockerized-test.properties @@ -0,0 +1 @@ +implementation-class = com.pedjak.gradle.plugins.dockerizedtest.DockerizedTestPlugin \ No newline at end of file