Add new Private ID DFCA binaries to build and release

Summary:
# Context
**Internal:** [Private ID DFCA one-pager](https://docs.google.com/document/d/1mwImSRCljqU66uXWJS7jL-dQOrYAR4aBZ3bpJ8quiJw) Advertisers want to be able to target their customers on our platform in a privacy-safe manner. We utilise Private ID with the existing DFCA solution to provide this.

We're adding a new game type `PRIVATE_ID_DFCA` whose inputs for both parties are csvs with the columns `user_pii, user_id_{party}`, and the output is a csv of `publisher_user_id, partner_user_id` which can then be fed to our existing DFCA APIs.
After running the existing PID stages, we need to run two new binaries to get the output in its final form:
- private_id_dfca_spine_combiner
- private_id_dfca_aggregator

# This Diff
Adds new binaries to build and release scripts. We can run a full e2e cloud test at this point.

# This Stack
- Add spine combiner binary
- Add aggregation binary
- Add aggregation stage
- Add Stageflow
- **Add game type**

Reviewed By: jrodal98

Differential Revision: D40510157

fbshipit-source-id: 910aeb8684f6c1bbf7df654d787ab7e6a327c90d

binaries_out_lists/data_processing.txt

@@ -3,3 +3,4 @@ sharder_hashed_for_pid
 pid_preparer
 lift_id_combiner
 attribution_id_combiner
+private_id_dfca_id_combiner

binaries_out_lists/emp_games.txt

@@ -7,3 +7,4 @@ decoupled_aggregation_calculator
 pcf2_attribution_calculator
 pcf2_aggregation_calculator
 shard_aggregator
+private_id_dfca_aggregator

docker/data_processing/CMakeLists.txt

@@ -147,3 +147,18 @@ target_link_libraries(
   gflags
 )
 install(TARGETS gen_fake_data DESTINATION bin)
+
+# private_id_dfca id combiner
+add_executable(
+  private_id_dfca_id_combiner
+  "fbpcs/data_processing/private_id_dfca_id_combiner/PrivateIdDfcaIdSpineCombiner.cpp"
+  "fbpcs/data_processing/private_id_dfca_id_combiner/PrivateIdDfcaIdSpineCombinerOptions.cpp"
+  "fbpcs/data_processing/private_id_dfca_id_combiner/PrivateIdDfcaIdSpineFileCombiner.cpp"
+  "fbpcs/data_processing/private_id_dfca_id_combiner/PidPrivateIdDfcaIdCombiner.cpp"
+  "fbpcs/data_processing/private_id_dfca_id_combiner/MrPidPrivateIdDfcaIdCombiner.cpp"
+  "fbpcs/data_processing/private_id_dfca_id_combiner/PrivateIdDfcaStrategy.cpp")
+target_link_libraries(
+  private_id_dfca_id_combiner
+  idcombiner
+  perftools)
+install(TARGETS private_id_dfca_id_combiner DESTINATION bin)

docker/data_processing/Dockerfile.ubuntu

@@ -22,6 +22,7 @@ COPY fbpcs/data_processing/lift_id_combiner/ ./fbpcs/data_processing/lift_id_com
 COPY fbpcs/data_processing/pid_preparer/ ./fbpcs/data_processing/pid_preparer
 COPY fbpcs/data_processing/sharding/ ./fbpcs/data_processing/sharding
 COPY fbpcs/data_processing/load_testing_utils/ ./fbpcs/data_processing/load_testing_utils
+COPY fbpcs/data_processing/private_id_dfca_id_combiner/ ./fbpcs/data_processing/private_id_dfca_id_combiner
 
 RUN cmake . -DTHREADING=ON -DEMP_USE_RANDOM_DEVICE=ON
 RUN make && make install

docker/emp_games/CMakeLists.txt

@@ -176,3 +176,18 @@ target_link_libraries(
   empgamecommon
   perftools)
 install(TARGETS pcf2_shard_combiner DESTINATION bin)
+
+# private_id_dfca_aggregator
+file(GLOB private_id_dfca_aggregator_src
+  "fbpcs/emp_games/private_id_dfca_aggregator/**.cpp"
+  "fbpcs/emp_games/private_id_dfca_aggregator/**.h"
+  "fbpcs/emp_games/private_id_dfca_aggregator/util/**.h")
+list(FILTER private_id_dfca_aggregator_src EXCLUDE REGEX ".*Test.*")
+add_executable(
+  private_id_dfca_aggregator
+  ${private_id_dfca_aggregator_src})
+target_link_libraries(
+  private_id_dfca_aggregator
+  empgamecommon
+  perftools)
+install(TARGETS private_id_dfca_aggregator DESTINATION bin)

docker/emp_games/Dockerfile.ubuntu

@@ -20,6 +20,7 @@ COPY fbpcs/emp_games/attribution/ ./fbpcs/emp_games/attribution
 COPY fbpcs/emp_games/pcf2_attribution/ ./fbpcs/emp_games/pcf2_attribution
 COPY fbpcs/emp_games/pcf2_aggregation/ ./fbpcs/emp_games/pcf2_aggregation
 COPY fbpcs/emp_games/pcf2_shard_combiner/ ./fbpcs/emp_games/pcf2_shard_combiner
+COPY fbpcs/emp_games/private_id_dfca_aggregator/ ./fbpcs/emp_games/private_id_dfca_aggregator
 COPY fbpcs/emp_games/lift/ ./fbpcs/emp_games/lift
 COPY fbpcs/emp_games/common/ ./fbpcs/emp_games/common
 

extract-docker-binaries.sh

@@ -72,6 +72,7 @@ docker cp temp_container:/usr/local/bin/pcf2_attribution_calculator "$SCRIPT_DIR
 docker cp temp_container:/usr/local/bin/pcf2_aggregation_calculator "$SCRIPT_DIR/binaries_out/."
 docker cp temp_container:/usr/local/bin/shard_aggregator "$SCRIPT_DIR/binaries_out/."
 docker cp temp_container:/usr/local/bin/pcf2_shard_combiner "$SCRIPT_DIR/binaries_out/."
+docker cp temp_container:/usr/local/bin/private_id_dfca_aggregator "$SCRIPT_DIR/binaries_out/."
 docker rm -f temp_container
 fi
 
@@ -82,6 +83,7 @@ docker cp temp_container:/usr/local/bin/sharder_hashed_for_pid "$SCRIPT_DIR/bina
 docker cp temp_container:/usr/local/bin/pid_preparer "$SCRIPT_DIR/binaries_out/."
 docker cp temp_container:/usr/local/bin/lift_id_combiner "$SCRIPT_DIR/binaries_out/."
 docker cp temp_container:/usr/local/bin/attribution_id_combiner "$SCRIPT_DIR/binaries_out/."
+docker cp temp_container:/usr/local/bin/private_id_dfca_id_combiner "$SCRIPT_DIR/binaries_out/."
 docker rm -f temp_container
 fi
 

promote_scripts/promote_binaries.sh

@@ -29,6 +29,7 @@ binary_names=(
     'private_attribution/pcf2_aggregation'
     'private_attribution/shard-aggregator'
     'private_attribution/pcf2_shard-combiner'
+    'private_id_dfca/private_id_dfca_aggregator'
     'pid/private-id-client'
     'pid/private-id-server'
     'pid/private-id-multi-key-client'
@@ -38,6 +39,7 @@ binary_names=(
     'data_processing/pid_preparer'
     'data_processing/lift_id_combiner'
     'data_processing/attribution_id_combiner'
+    'data_processing/private_id_dfca_id_combiner'
     'validation/pc_pre_validation_cli'
 )
 

upload-binaries-to-s3-test.sh

@@ -40,6 +40,7 @@ pcf2_attribution="$attribution_repo/pcf2_attribution/${TAG}/pcf2_attribution"
 pcf2_aggregation="$attribution_repo/pcf2_aggregation/${TAG}/pcf2_aggregation"
 shard_aggregator_package="$attribution_repo/shard-aggregator/${TAG}/shard-aggregator"
 pcf2_shard_combiner_package="$attribution_repo/pcf2_shard-combiner/${TAG}/pcf2_shard-combiner"
+private_id_dfca_aggregator_package="s3://$one_docker_repo/private_id_dfca/private_id_dfca_aggregator/${TAG}/private_id_dfca_aggregator"
 data_processing_repo="s3://$one_docker_repo/data_processing"
 private_id_repo="s3://$one_docker_repo/pid"
 validation_repo="s3://$one_docker_repo/validation"
@@ -55,6 +56,7 @@ aws s3 cp pcf2_attribution_calculator "$pcf2_attribution"
 aws s3 cp pcf2_aggregation_calculator "$pcf2_aggregation"
 aws s3 cp shard_aggregator "$shard_aggregator_package"
 aws s3 cp pcf2_shard_combiner "$pcf2_shard_combiner_package"
+aws s3 cp private_id_dfca_aggregator "$private_id_dfca_aggregator_package"
 cd .. || exit
 fi
 
@@ -66,6 +68,7 @@ aws s3 cp sharder_hashed_for_pid "$data_processing_repo/sharder_hashed_for_pid/$
 aws s3 cp pid_preparer "$data_processing_repo/pid_preparer/${TAG}/pid_preparer"
 aws s3 cp lift_id_combiner "$data_processing_repo/lift_id_combiner/${TAG}/lift_id_combiner"
 aws s3 cp attribution_id_combiner "$data_processing_repo/attribution_id_combiner/${TAG}/attribution_id_combiner"
+aws s3 cp private_id_dfca_id_combiner "$data_processing_repo/private_id_dfca_id_combiner/${TAG}/private_id_dfca_id_combiner"
 fi
 
 if [ "$PACKAGE" = "pid" ]; then

upload-binaries-to-s3.sh

@@ -40,6 +40,7 @@ pcf2_attribution="$attribution_repo/pcf2_attribution/${TAG}/pcf2_attribution"
 pcf2_aggregation="$attribution_repo/pcf2_aggregation/${TAG}/pcf2_aggregation"
 shard_aggregator_package="$attribution_repo/shard-aggregator/${TAG}/shard-aggregator"
 pcf2_shard_combiner_package="$attribution_repo/pcf2_shard-combiner/${TAG}/pcf2_shard-combiner"
+private_id_dfca_aggregator_package="s3://$one_docker_repo/private_id_dfca/private_id_dfca_aggregator/${TAG}/private_id_dfca_aggregator"
 data_processing_repo="s3://$one_docker_repo/data_processing"
 private_id_repo="s3://$one_docker_repo/pid"
 validation_repo="s3://$one_docker_repo/validation"
@@ -55,6 +56,7 @@ aws s3 cp pcf2_attribution_calculator "$pcf2_attribution"
 aws s3 cp pcf2_aggregation_calculator "$pcf2_aggregation"
 aws s3 cp shard_aggregator "$shard_aggregator_package"
 aws s3 cp pcf2_shard_combiner "$pcf2_shard_combiner_package"
+aws s3 cp private_id_dfca_aggregator "$private_id_dfca_aggregator_package"
 cd .. || exit
 fi
 
@@ -66,6 +68,7 @@ aws s3 cp sharder_hashed_for_pid "$data_processing_repo/sharder_hashed_for_pid/$
 aws s3 cp pid_preparer "$data_processing_repo/pid_preparer/${TAG}/pid_preparer"
 aws s3 cp lift_id_combiner "$data_processing_repo/lift_id_combiner/${TAG}/lift_id_combiner"
 aws s3 cp attribution_id_combiner "$data_processing_repo/attribution_id_combiner/${TAG}/attribution_id_combiner"
+aws s3 cp private_id_dfca_id_combiner "$data_processing_repo/private_id_dfca_id_combiner/${TAG}/private_id_dfca_id_combiner"
 fi
 
 if [ "$PACKAGE" = "pid" ]; then