Skip to content

Commit

Permalink
ActionInputFileCache: move getMetadata to a new super-interface
Browse files Browse the repository at this point in the history
Update the callers that only need getMetadata to use the new interface.

PiperOrigin-RevId: 167992239
  • Loading branch information
ulfjack authored and philwo committed Sep 11, 2017
1 parent f322ba7 commit 5c3467f
Show file tree
Hide file tree
Showing 7 changed files with 57 additions and 34 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -419,13 +419,13 @@ protected void deleteOutput(Artifact output) throws IOException {
* checking, this method must be called.
*/
protected void checkInputsForDirectories(
EventHandler eventHandler, ActionInputFileCache metadataHandler) throws ExecException {
EventHandler eventHandler, MetadataProvider metadataProvider) throws ExecException {
// Report "directory dependency checking" warning only for non-generated directories (generated
// ones will be reported earlier).
for (Artifact input : getMandatoryInputs()) {
// Assume that if the file did not exist, we would not have gotten here.
try {
if (input.isSourceArtifact() && !metadataHandler.getMetadata(input).isFile()) {
if (input.isSourceArtifact() && !metadataProvider.getMetadata(input).isFile()) {
eventHandler.handle(Event.warn(getOwner().getLocation(), "input '"
+ input.prettyPrint() + "' to " + getOwner().getLabel()
+ " is a directory; dependency checking of directories is unsound"));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,8 @@
// limitations under the License.
package com.google.devtools.build.lib.actions;

import com.google.devtools.build.lib.actions.cache.Metadata;
import com.google.devtools.build.lib.vfs.Path;
import com.google.protobuf.ByteString;
import java.io.IOException;
import javax.annotation.Nullable;
import javax.annotation.concurrent.ThreadSafe;

Expand All @@ -26,26 +24,7 @@
* NOTE: Implementations must be thread safe.
*/
@ThreadSafe
public interface ActionInputFileCache {
/**
* Returns digest for the given artifact. This digest is current as of some time t >= the start of
* the present build. If the artifact is an output of an action that already executed at time p,
* then t >= p. Aside from these properties, t can be any value and may vary arbitrarily across
* calls.
*
* The return value is owned by the cache and must not be modified.
*
* @param input the input to retrieve the digest for
* @return the artifact's digest or null if digest cannot be obtained (due to artifact
* non-existence, lookup errors, or any other reason)
*
* @throws DigestOfDirectoryException in case {@code input} is a directory.
* @throws IOException If the file cannot be digested.
*
*/
@Nullable
Metadata getMetadata(ActionInput input) throws IOException;

public interface ActionInputFileCache extends MetadataProvider {
/**
* Checks if the file is available locally, based on the assumption that previous operations on
* the ActionInputFileCache would have created a cache entry for it.
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
// Copyright 2017 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.actions;

import com.google.devtools.build.lib.actions.cache.Metadata;
import java.io.IOException;
import javax.annotation.Nullable;

/**
* The interface for Action inputs metadata (Digest and size).
*
* NOTE: Implementations must be thread safe.
*/
public interface MetadataProvider {
/**
* Returns digest for the given artifact. This digest is current as of some time t >= the start of
* the present build. If the artifact is an output of an action that already executed at time p,
* then t >= p. Aside from these properties, t can be any value and may vary arbitrarily across
* calls.
*
* The return value is owned by the cache and must not be modified.
*
* @param input the input to retrieve the digest for
* @return the artifact's digest or null if digest cannot be obtained (due to artifact
* non-existence, lookup errors, or any other reason)
*
* @throws DigestOfDirectoryException in case {@code input} is a directory.
* @throws IOException If the file cannot be digested.
*
*/
@Nullable
Metadata getMetadata(ActionInput input) throws IOException;
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
import com.google.common.base.Throwables;
import com.google.common.io.ByteStreams;
import com.google.devtools.build.lib.actions.ActionInput;
import com.google.devtools.build.lib.actions.ActionInputFileCache;
import com.google.devtools.build.lib.actions.MetadataProvider;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.remoteexecution.v1test.Digest;
import com.google.protobuf.ByteString;
Expand Down Expand Up @@ -137,12 +137,12 @@ public Chunker(Path file, int chunkSize) throws IOException {
}, Digests.computeDigest(file), chunkSize);
}

public Chunker(ActionInput actionInput, ActionInputFileCache inputCache, Path execRoot) throws
public Chunker(ActionInput actionInput, MetadataProvider inputCache, Path execRoot) throws
IOException{
this(actionInput, inputCache, execRoot, getDefaultChunkSize());
}

public Chunker(ActionInput actionInput, ActionInputFileCache inputCache, Path execRoot,
public Chunker(ActionInput actionInput, MetadataProvider inputCache, Path execRoot,
int chunkSize)
throws IOException {
this(() -> {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

import com.google.common.hash.HashCode;
import com.google.devtools.build.lib.actions.ActionInput;
import com.google.devtools.build.lib.actions.ActionInputFileCache;
import com.google.devtools.build.lib.actions.MetadataProvider;
import com.google.devtools.build.lib.actions.cache.DigestUtils;
import com.google.devtools.build.lib.actions.cache.Metadata;
import com.google.devtools.build.lib.actions.cache.VirtualActionInput;
Expand Down Expand Up @@ -102,7 +102,7 @@ public static Digest buildDigest(String hexHash, long size) {
return Digest.newBuilder().setHash(hexHash).setSizeBytes(size).build();
}

public static Digest getDigestFromInputCache(ActionInput input, ActionInputFileCache cache)
public static Digest getDigestFromInputCache(ActionInput input, MetadataProvider cache)
throws IOException {
Metadata metadata = cache.getMetadata(input);
return buildDigest(metadata.getDigest(), metadata.getSize());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,9 @@
import com.google.common.util.concurrent.ListeningScheduledExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.devtools.build.lib.actions.ActionInput;
import com.google.devtools.build.lib.actions.ActionInputFileCache;
import com.google.devtools.build.lib.actions.EnvironmentalExecException;
import com.google.devtools.build.lib.actions.ExecException;
import com.google.devtools.build.lib.actions.MetadataProvider;
import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
import com.google.devtools.build.lib.remote.Digests.ActionKey;
import com.google.devtools.build.lib.remote.TreeNodeRepository.TreeNode;
Expand Down Expand Up @@ -156,7 +156,7 @@ public void ensureInputsPresent(
uploadBlob(command.toByteArray());
if (!missingActionInputs.isEmpty()) {
List<Chunker> inputsToUpload = new ArrayList<>();
ActionInputFileCache inputFileCache = repository.getInputFileCache();
MetadataProvider inputFileCache = repository.getInputFileCache();
for (ActionInput actionInput : missingActionInputs) {
inputsToUpload.add(new Chunker(actionInput, inputFileCache, execRoot));
}
Expand Down Expand Up @@ -374,7 +374,7 @@ private Digest uploadFileContents(Path file) throws IOException, InterruptedExce
*
* @return The key for fetching the file contents blob from cache.
*/
Digest uploadFileContents(ActionInput input, Path execRoot, ActionInputFileCache inputCache)
Digest uploadFileContents(ActionInput input, Path execRoot, MetadataProvider inputCache)
throws IOException, InterruptedException {
Digest digest = Digests.getDigestFromInputCache(input, inputCache);
ImmutableSet<Digest> missing = getMissingDigests(ImmutableList.of(digest));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,9 @@
package com.google.devtools.build.lib.remote;

import com.google.devtools.build.lib.actions.ActionInput;
import com.google.devtools.build.lib.actions.ActionInputFileCache;
import com.google.devtools.build.lib.actions.EnvironmentalExecException;
import com.google.devtools.build.lib.actions.ExecException;
import com.google.devtools.build.lib.actions.MetadataProvider;
import com.google.devtools.build.lib.actions.cache.VirtualActionInput;
import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
import com.google.devtools.build.lib.remote.Digests.ActionKey;
Expand Down Expand Up @@ -99,7 +99,7 @@ private Digest uploadFileContents(Path file) throws IOException, InterruptedExce
}

private Digest uploadFileContents(
ActionInput input, Path execRoot, ActionInputFileCache inputCache)
ActionInput input, Path execRoot, MetadataProvider inputCache)
throws IOException, InterruptedException {
// This unconditionally reads the whole file into memory first!
if (input instanceof VirtualActionInput) {
Expand Down

0 comments on commit 5c3467f

Please sign in to comment.