Skip to content

Commit

Permalink
Use the RecordingInputStream from the WorkerSpawnStrategy in the Work…
Browse files Browse the repository at this point in the history
…erTestStrategy as well, to better log errors.

RELNOTES: None
PiperOrigin-RevId: 154080821
  • Loading branch information
kush-c authored and vladmos committed Apr 25, 2017
1 parent 2195f25 commit 2968bcf
Show file tree
Hide file tree
Showing 3 changed files with 131 additions and 94 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
// Copyright 2017 The Bazel Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.devtools.build.lib.worker;

import java.io.ByteArrayOutputStream;
import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.nio.charset.Charset;
import java.util.regex.Pattern;

/**
* An input stream filter that records the first X bytes read from its wrapped stream.
*
* <p>The number bytes to record can be set via {@link #startRecording(int)}}, which also discards
* any already recorded data. The recorded data can be retrieved via {@link
* #getRecordedDataAsString(Charset)}.
*/
final class RecordingInputStream extends FilterInputStream {
private static final Pattern NON_PRINTABLE_CHARS =
Pattern.compile("[^\\p{Print}\\t\\r\\n]", Pattern.UNICODE_CHARACTER_CLASS);

private ByteArrayOutputStream recordedData;
private int maxRecordedSize;

RecordingInputStream(InputStream in) {
super(in);
}

/**
* Returns the maximum number of bytes that can still be recorded in our buffer (but not more
* than {@code size}).
*/
private int getRecordableBytes(int size) {
if (recordedData == null) {
return 0;
}
return Math.min(maxRecordedSize - recordedData.size(), size);
}

@Override
public int read() throws IOException {
int bytesRead = super.read();
if (getRecordableBytes(bytesRead) > 0) {
recordedData.write(bytesRead);
}
return bytesRead;
}

@Override
public int read(byte[] b) throws IOException {
int bytesRead = super.read(b);
int recordableBytes = getRecordableBytes(bytesRead);
if (recordableBytes > 0) {
recordedData.write(b, 0, recordableBytes);
}
return bytesRead;
}

@Override
public int read(byte[] b, int off, int len) throws IOException {
int bytesRead = super.read(b, off, len);
int recordableBytes = getRecordableBytes(bytesRead);
if (recordableBytes > 0) {
recordedData.write(b, off, recordableBytes);
}
return bytesRead;
}

public void startRecording(int maxSize) {
recordedData = new ByteArrayOutputStream(maxSize);
maxRecordedSize = maxSize;
}

/**
* Reads whatever remaining data is available on the input stream if we still have space left in
* the recording buffer, in order to maximize the usefulness of the recorded data for the
* caller.
*/
public void readRemaining() {
try {
byte[] dummy = new byte[getRecordableBytes(available())];
read(dummy);
} catch (IOException e) {
// Ignore.
}
}

/**
* Returns the recorded data as a string, where non-printable characters are replaced with a '?'
* symbol.
*/
public String getRecordedDataAsString(Charset charsetName) throws UnsupportedEncodingException {
String recordedString = recordedData.toString(charsetName.name());
return NON_PRINTABLE_CHARS.matcher(recordedString).replaceAll("?").trim();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -53,12 +53,8 @@
import com.google.devtools.build.lib.worker.WorkerProtocol.WorkRequest;
import com.google.devtools.build.lib.worker.WorkerProtocol.WorkResponse;
import com.google.protobuf.ByteString;
import java.io.ByteArrayOutputStream;
import java.io.FilterInputStream;
import com.google.protobuf.InvalidProtocolBufferException;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
Expand All @@ -78,93 +74,6 @@
)
public final class WorkerSpawnStrategy implements SandboxedSpawnActionContext {

/**
* An input stream filter that records the first X bytes read from its wrapped stream.
*
* <p>The number bytes to record can be set via {@link #startRecording(int)}}, which also discards
* any already recorded data. The recorded data can be retrieved via {@link
* #getRecordedDataAsString(Charset)}.
*/
private static final class RecordingInputStream extends FilterInputStream {
private static final Pattern NON_PRINTABLE_CHARS =
Pattern.compile("[^\\p{Print}\\t\\r\\n]", Pattern.UNICODE_CHARACTER_CLASS);

private ByteArrayOutputStream recordedData;
private int maxRecordedSize;

protected RecordingInputStream(InputStream in) {
super(in);
}

/**
* Returns the maximum number of bytes that can still be recorded in our buffer (but not more
* than {@code size}).
*/
private int getRecordableBytes(int size) {
if (recordedData == null) {
return 0;
}
return Math.min(maxRecordedSize - recordedData.size(), size);
}

@Override
public int read() throws IOException {
int bytesRead = super.read();
if (getRecordableBytes(bytesRead) > 0) {
recordedData.write(bytesRead);
}
return bytesRead;
}

@Override
public int read(byte[] b) throws IOException {
int bytesRead = super.read(b);
int recordableBytes = getRecordableBytes(bytesRead);
if (recordableBytes > 0) {
recordedData.write(b, 0, recordableBytes);
}
return bytesRead;
}

@Override
public int read(byte[] b, int off, int len) throws IOException {
int bytesRead = super.read(b, off, len);
int recordableBytes = getRecordableBytes(bytesRead);
if (recordableBytes > 0) {
recordedData.write(b, off, recordableBytes);
}
return bytesRead;
}

public void startRecording(int maxSize) {
recordedData = new ByteArrayOutputStream(maxSize);
maxRecordedSize = maxSize;
}

/**
* Reads whatever remaining data is available on the input stream if we still have space left in
* the recording buffer, in order to maximize the usefulness of the recorded data for the
* caller.
*/
public void readRemaining() {
try {
byte[] dummy = new byte[getRecordableBytes(available())];
read(dummy);
} catch (IOException e) {
// Ignore.
}
}

/**
* Returns the recorded data as a string, where non-printable characters are replaced with a '?'
* symbol.
*/
public String getRecordedDataAsString(Charset charsetName) throws UnsupportedEncodingException {
String recordedString = recordedData.toString(charsetName.name());
return NON_PRINTABLE_CHARS.matcher(recordedString).replaceAll("?").trim();
}
}

public static final String ERROR_MESSAGE_PREFIX =
"Worker strategy cannot execute this %s action, ";
public static final String REASON_NO_FLAGFILE =
Expand Down Expand Up @@ -387,7 +296,7 @@ private WorkResponse execInWorker(
recordingStream.startRecording(4096);
try {
response = WorkResponse.parseDelimitedFrom(recordingStream);
} catch (IOException e2) {
} catch (InvalidProtocolBufferException e2) {
// If protobuf couldn't parse the response, try to print whatever the failing worker wrote
// to stdout - it's probably a stack trace or some kind of error message that will help the
// user figure out why the compiler is failing.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

package com.google.devtools.build.lib.worker;

import com.google.common.base.Charsets;
import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
Expand All @@ -40,6 +41,7 @@
import com.google.devtools.build.lib.worker.WorkerProtocol.WorkRequest;
import com.google.devtools.build.lib.worker.WorkerProtocol.WorkResponse;
import com.google.devtools.common.options.OptionsClassProvider;
import com.google.protobuf.InvalidProtocolBufferException;
import java.io.IOException;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -152,7 +154,23 @@ private TestResultData execInWorker(
request.writeDelimitedTo(worker.getOutputStream());
worker.getOutputStream().flush();

WorkResponse response = WorkResponse.parseDelimitedFrom(worker.getInputStream());
RecordingInputStream recordingStream = new RecordingInputStream(worker.getInputStream());
recordingStream.startRecording(4096);
WorkResponse response;
try {
response = WorkResponse.parseDelimitedFrom(recordingStream);
} catch (InvalidProtocolBufferException e) {
// If protobuf couldn't parse the response, try to print whatever the failing worker wrote
// to stdout - it's probably a stack trace or some kind of error message that will help the
// user figure out why the compiler is failing.
recordingStream.readRemaining();
String data = recordingStream.getRecordedDataAsString(Charsets.UTF_8);
executor
.getEventHandler()
.handle(Event.warn("Worker process returned an unparseable WorkResponse:\n" + data));
throw e;
}

actionExecutionContext.getFileOutErr().getErrorStream().write(
response.getOutputBytes().toByteArray());

Expand Down

0 comments on commit 2968bcf

Please sign in to comment.