chore: Update airflow config

airflow/utils/common.py

@@ -58,10 +58,10 @@ def add_env_suffix(config: dict) -> dict:
         dict: The configuration dictionary with environment suffix added to table names.
     """
     env_suffix = os.getenv("ENV_SUBFIX", config.get("env", ""))
-    if "gcp" in config and "table_name" in config["gcp"]:
-        for key, value in config["gcp"]["table_name"].items():
+    if "gcp" in config and "table" in config["gcp"]:
+        for key, value in config["gcp"]["table"].items():
             if isinstance(value, str) and not value.startswith("ENV_"):
-                config["gcp"]["table_name"][key] = f"{env_suffix}-{value}"
+                config["gcp"]["table"][key] = f"{env_suffix}-{value}"
     return config
 
 

airflow/utils/config.yml

@@ -4,15 +4,17 @@ gcp:
     raw: ENV_GCP_GCS_RAW_BUCKET
     processed: ENV_GCP_GCS_PROCESSED_BUCKET
     archive: ENV_GCP_GCS_ARCHIVE_BUCKET
-  blob_name:
+  blob:
     gmaps:
+      prefix: "gmaps"
+      places: "gmaps/places/*.jsonl"
       reviews: "gmaps/detailed-reviews/*.parquet"
-      places: "gmaps/attractions/*.parquet"
   bigquery:
+    src_dataset: ENV_BIGQUERY_SRC_DATASET
     ods_dataset: ENV_BIGQUERY_ODS_DATASET
     dim_dataset: ENV_BIGQUERY_DIM_DATASET
     fact_dataset: ENV_BIGQUERY_FACT_DATASET
     mart_dataset: ENV_BIGQUERY_FACT_DATASET
-  table_name:
-    gmaps_places: "gmaps_places"
-    gmaps_reviews: "gmaps_reviews"
+  table:
+    gmaps-places: "gmaps-places"
+    gmaps-reviews: "gmaps-reviews"