Programatically add rust config to vendored protos (pantsbuild#7000)

I'm about to add a second build script which will use prost for its
codegen, and which needs the pristine protos, so instead of checking
them in altered, alter them on the fly specifically for protoc_grpcio.

This also makes it easier to pull in updates.

3rdparty/protobuf/bazelbuild_remote-apis/README.md

@@ -1,12 +1,3 @@
 This is a dump of the .proto files from https://github.com/bazelbuild/remote-apis directory build.
 
 This dump was taken at git sha cbf6ada7f5b2a0ce14646bf983d03b49118f0ec8.
-
-The following script was run to enable Bytes fields for Rust:
-```
-sed -i '' '/^package /a\
-\
-import "rustproto.proto";\
-option (rustproto.carllerche_bytes_for_bytes_all) = true;\
-' build/bazel/remote/execution/v2/remote_execution.proto
-```

3rdparty/protobuf/bazelbuild_remote-apis/build/bazel/remote/execution/v2/remote_execution.proto

@@ -16,9 +16,6 @@ syntax = "proto3";
 
 package build.bazel.remote.execution.v2;
 
-import "rustproto.proto";
-option (rustproto.carllerche_bytes_for_bytes_all) = true;
-
 import "google/api/annotations.proto";
 import "google/longrunning/operations.proto";
 import "google/protobuf/duration.proto";

3rdparty/protobuf/googleapis/README.md

@@ -3,12 +3,3 @@ This is a dump of the .proto files from https://github.com/googleapis/googleapis
 This dump was taken at git sha e17dbfb19652240490cae8adeb89991d13cf9df7.
 
 It is a selective view of only the protos we actually need.
-
-The following script was run to enable Bytes fields for Rust:
-```
-sed -i '' '/^package /a\
-\
-import "rustproto.proto";\
-option (rustproto.carllerche_bytes_for_bytes_all) = true;\
-' {google/devtools/remoteexecution/v1test/remote_execution.proto,google/bytestream/bytestream.proto,google/rpc/{code,error_details,status}.proto,google/longrunning/operations.proto}
-```

3rdparty/protobuf/googleapis/google/bytestream/bytestream.proto

@@ -16,9 +16,6 @@ syntax = "proto3";
 
 package google.bytestream;
 
-import "rustproto.proto";
-option (rustproto.carllerche_bytes_for_bytes_all) = true;
-
 import "google/api/annotations.proto";
 import "google/protobuf/wrappers.proto";
 

3rdparty/protobuf/googleapis/google/longrunning/operations.proto

@@ -16,9 +16,6 @@ syntax = "proto3";
 
 package google.longrunning;
 
-import "rustproto.proto";
-option (rustproto.carllerche_bytes_for_bytes_all) = true;
-
 import "google/api/annotations.proto";
 import "google/protobuf/any.proto";
 import "google/protobuf/empty.proto";

3rdparty/protobuf/googleapis/google/rpc/code.proto

@@ -16,9 +16,6 @@ syntax = "proto3";
 
 package google.rpc;
 
-import "rustproto.proto";
-option (rustproto.carllerche_bytes_for_bytes_all) = true;
-
 option go_package = "google.golang.org/genproto/googleapis/rpc/code;code";
 option java_multiple_files = true;
 option java_outer_classname = "CodeProto";

3rdparty/protobuf/googleapis/google/rpc/error_details.proto

@@ -16,9 +16,6 @@ syntax = "proto3";
 
 package google.rpc;
 
-import "rustproto.proto";
-option (rustproto.carllerche_bytes_for_bytes_all) = true;
-
 import "google/protobuf/duration.proto";
 
 option go_package = "google.golang.org/genproto/googleapis/rpc/errdetails;errdetails";

3rdparty/protobuf/googleapis/google/rpc/status.proto

@@ -16,9 +16,6 @@ syntax = "proto3";
 
 package google.rpc;
 
-import "rustproto.proto";
-option (rustproto.carllerche_bytes_for_bytes_all) = true;
-
 import "google/protobuf/any.proto";
 
 option go_package = "google.golang.org/genproto/googleapis/rpc/status;status";

src/rust/engine/process_execution/bazel_protos/Cargo.toml

@@ -16,3 +16,5 @@ protobuf = { version = "2.0.4", features = ["with-bytes"] }
 build_utils = { path = "../../build_utils" }
 grpcio-compiler = "0.3"
 protoc-grpcio = "0.2"
+tempfile = "3"
+walkdir = "2"

src/rust/engine/process_execution/bazel_protos/build.rs

@@ -2,7 +2,7 @@ use protoc_grpcio;
 
 use std::fs::File;
 use std::io::Write;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 
 use build_utils::BuildRoot;
 
@@ -14,6 +14,9 @@ fn main() {
     thirdpartyprotobuf.to_str().unwrap()
   );
 
+  let amended_proto_root =
+    add_rustproto_header(&thirdpartyprotobuf).expect("Error adding proto bytes header");
+
   let gen_dir = PathBuf::from("src/gen");
 
   // Re-gen if, say, someone does a git clean on the gen dir but not the target dir. This ensures
@@ -31,8 +34,7 @@ fn main() {
       "google/protobuf/empty.proto",
     ],
     &[
-      thirdpartyprotobuf.join("bazelbuild_remote-apis"),
-      thirdpartyprotobuf.join("googleapis"),
+      amended_proto_root.path().to_owned(),
       thirdpartyprotobuf.join("standard"),
       thirdpartyprotobuf.join("rust-protobuf"),
     ],
@@ -64,3 +66,43 @@ fn main() {
     .and_then(|mut f| f.write_all(contents.as_bytes()))
     .expect("Failed to write mod.rs")
 }
+
+const EXTRA_HEADER: &'static str = r#"import "rustproto.proto";
+option (rustproto.carllerche_bytes_for_bytes_all) = true;
+"#;
+
+///
+/// Copies protos from thirdpartyprotobuf, adds a header to make protoc_grpcio uses Bytes instead
+/// of Vec<u8>s, and rewrites them into a temporary directory
+///
+fn add_rustproto_header(thirdpartyprotobuf: &Path) -> Result<tempfile::TempDir, String> {
+  let amended_proto_root = tempfile::TempDir::new().unwrap();
+  for f in &["bazelbuild_remote-apis", "googleapis"] {
+    let src_root = thirdpartyprotobuf.join(f);
+    for entry in walkdir::WalkDir::new(&src_root)
+      .into_iter()
+      .filter_map(|entry| entry.ok())
+      .filter(|entry| entry.file_type().is_file())
+      .filter(|entry| entry.file_name().to_string_lossy().ends_with(".proto"))
+    {
+      let dst = amended_proto_root
+        .path()
+        .join(entry.path().strip_prefix(&src_root).unwrap());
+      std::fs::create_dir_all(dst.parent().unwrap())
+        .map_err(|err| format!("Error making dir in temp proto root: {}", err))?;
+      let original = std::fs::read_to_string(entry.path())
+        .map_err(|err| format!("Error reading proto {}: {}", entry.path().display(), err))?;
+      let mut copy = String::with_capacity(original.len() + EXTRA_HEADER.len());
+      for line in original.lines() {
+        copy += line;
+        copy += "\n";
+        if line.starts_with("package ") {
+          copy += EXTRA_HEADER
+        }
+      }
+      std::fs::write(&dst, copy.as_bytes())
+        .map_err(|err| format!("Error writing {}: {}", dst.display(), err))?;
+    }
+  }
+  Ok(amended_proto_root)
+}