Fix: rename all binaries where cluster was remote to coiled;

aodn · Nov 21, 2024 · 7a999bd · 7a999bd
1 parent d9a238d
commit 7a999bd
Show file tree

Hide file tree

Showing 55 changed files with 69 additions and 69 deletions.
diff --git a/aodn_cloud_optimised/bin/argo.py b/aodn_cloud_optimised/bin/argo.py
@@ -21,7 +21,7 @@ def main():
     #     # "--clear-existing-data",
     #     "--force-previous-parquet-deletion",
     #     "--cluster-mode",
-    #     "remote",
+    #     "coiled",
     # ]
     #
     # # Run the command
@@ -38,7 +38,7 @@ def main():
         "argo.json",
         "--force-previous-parquet-deletion",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
     subprocess.run(command, check=True)
 
@@ -56,7 +56,7 @@ def main():
         "argo.json",
         "--force-previous-parquet-deletion",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
     subprocess.run(command, check=True)
 
@@ -73,6 +73,6 @@ def main():
         "argo.json",
         "--force-previous-parquet-deletion",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
     subprocess.run(command, check=True)
diff --git a/aodn_cloud_optimised/bin/autonomous_underwater_vehicle.py b/aodn_cloud_optimised/bin/autonomous_underwater_vehicle.py
@@ -895,7 +895,7 @@ def main():
         "autonomous_underwater_vehicle.json",
         "--clear-existing-data",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
 
     # Run the command

diff --git a/aodn_cloud_optimised/bin/create_dataset_config.py b/aodn_cloud_optimised/bin/create_dataset_config.py
@@ -204,7 +204,7 @@ def main():
         "{dataset_json}",
         "--clear-existing-data",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
 
     # Run the command
@@ -678,7 +678,7 @@ def main():
     dataset_config["metadata_uuid"] = args.uuid
     dataset_config["logger_name"] = args.dataset_name
     dataset_config["cloud_optimised_format"] = args.cloud_format
-    dataset_config["cluster_options"] = {
+    dataset_config["coiled_cluster_options"] = {
         "n_workers": [1, 20],
         "scheduler_vm_types": "t3.small",
         "worker_vm_types": "t3.medium",

diff --git a/aodn_cloud_optimised/bin/generic_cloud_optimised_creation.py b/aodn_cloud_optimised/bin/generic_cloud_optimised_creation.py
@@ -9,13 +9,13 @@
 Usage Examples:
   generic_cloud_optimised_creation --paths 'IMOS/ANMN/NSW' 'IMOS/ANMN/PA'
   --filters '_hourly-timeseries_' 'FV02' --dataset-config 'mooring_hourly_timeseries_delayed_qc.json'
-  --clear-existing-data --cluster-mode 'remote'
+  --clear-existing-data --cluster-mode 'coiled'
 
   generic_cloud_optimised_creation --paths 'IMOS/ANMN/NSW' 'IMOS/ANMN/QLD'
   --dataset-config 'mooring_ctd_delayed_qc.json'
 
   generic_cloud_optimised_creation --paths 'IMOS/ACORN/gridded_1h-avg-current-map_QC/TURQ/2024'
-  --dataset-config 'radar_TurquoiseCoast_velocity_hourly_average_delayed_qc.json' --clear-existing-data --cluster-mode 'remote'
+  --dataset-config 'radar_TurquoiseCoast_velocity_hourly_average_delayed_qc.json' --clear-existing-data --cluster-mode 'coiled'
 
 Arguments:
   --paths: List of S3 paths to process. Example: 'IMOS/ANMN/NSW' 'IMOS/ANMN/PA'
@@ -25,7 +25,7 @@
   --clear-existing-data: Flag to clear existing data. Default is False.
   --force-previous-parquet-deletion: Flag to force the search of previous equivalent parquet file created. Much slower.
                                      Default is False. Only for Parquet processing.
-  --cluster-mode: Cluster mode to use. Options: 'local' or 'remote'. Default is 'local'.
+  --cluster-mode: Cluster mode to use. Options: 'local' or 'coiled'. Default is 'local'.
   --optimised-bucket-name: Bucket name where cloud optimised object will be created. Default is the value of
                            BUCKET_OPTIMISED_DEFAULT from the config.
   --root-prefix-cloud-optimised-path: Prefix value for the root location of the cloud optimised objects. Default is the
@@ -51,9 +51,9 @@ def main():
     parser = argparse.ArgumentParser(
         description="Process S3 paths and create cloud-optimized datasets.",
         epilog="Examples:\n"
-        "  generic_cloud_optimised_creation --paths 'IMOS/ANMN/NSW' 'IMOS/ANMN/PA' --filters '_hourly-timeseries_' 'FV02' --dataset-config 'mooring_hourly_timeseries_delayed_qc.json' --clear-existing-data --cluster-mode 'remote'\n"
+        "  generic_cloud_optimised_creation --paths 'IMOS/ANMN/NSW' 'IMOS/ANMN/PA' --filters '_hourly-timeseries_' 'FV02' --dataset-config 'mooring_hourly_timeseries_delayed_qc.json' --clear-existing-data --cluster-mode 'coiled'\n"
         "  generic_cloud_optimised_creation --paths 'IMOS/ANMN/NSW' 'IMOS/ANMN/QLD' --dataset-config 'mooring_ctd_delayed_qc.json'\n"
-        "  generic_cloud_optimised_creation --paths 'IMOS/ACORN/gridded_1h-avg-current-map_QC/TURQ/2024' --dataset-config 'radar_TurquoiseCoast_velocity_hourly_average_delayed_qc.json' --clear-existing-data --cluster-mode 'remote'\n",
+        "  generic_cloud_optimised_creation --paths 'IMOS/ACORN/gridded_1h-avg-current-map_QC/TURQ/2024' --dataset-config 'radar_TurquoiseCoast_velocity_hourly_average_delayed_qc.json' --clear-existing-data --cluster-mode 'coiled'\n",
         formatter_class=argparse.RawTextHelpFormatter,
     )
 
@@ -91,13 +91,13 @@ def main():
         "Only for Parquet processing.",
     )
 
-    cluster_options = [mode.value for mode in clusterLib.ClusterMode]
+    coiled_cluster_options = [mode.value for mode in clusterLib.ClusterMode]
     parser.add_argument(
         "--cluster-mode",
         # type=clusterLib.parse_cluster_mode,
         default=clusterLib.ClusterMode.NONE.value,
-        choices=cluster_options,
-        help="Cluster mode to use. Options: 'local' or 'remote'. Default is None.",
+        choices=coiled_cluster_options,
+        help="Cluster mode to use. Options: 'local' or 'coiled'. Default is None.",
     )
 
     parser.add_argument(

diff --git a/aodn_cloud_optimised/bin/model_sea_level_anomaly_gridded_realtime.py b/aodn_cloud_optimised/bin/model_sea_level_anomaly_gridded_realtime.py
@@ -11,7 +11,7 @@ def main():
         "model_sea_level_anomaly_gridded_realtime.json",
         "--clear-existing-data",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
 
     # Run the command

diff --git a/aodn_cloud_optimised/bin/mooring_ctd_delayed_qc.py b/aodn_cloud_optimised/bin/mooring_ctd_delayed_qc.py
@@ -18,7 +18,7 @@ def main():
         "mooring_ctd_delayed_qc.json",
         "--clear-existing-data",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
 
     # Run the command

diff --git a/aodn_cloud_optimised/bin/mooring_hourly_timeseries_delayed_qc.py b/aodn_cloud_optimised/bin/mooring_hourly_timeseries_delayed_qc.py
@@ -18,7 +18,7 @@ def main():
         "mooring_hourly_timeseries_delayed_qc.json",
         "--clear-existing-data",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
 
     # Run the command

diff --git a/aodn_cloud_optimised/bin/mooring_satellite_altimetry_calibration_validation.py b/aodn_cloud_optimised/bin/mooring_satellite_altimetry_calibration_validation.py
@@ -14,7 +14,7 @@ def main():
         "mooring_satellite_altimetry_calibration_validation.json",
         "--clear-existing-data",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
 
     # Run the command

diff --git a/aodn_cloud_optimised/bin/mooring_temperature_logger_delayed_qc.py b/aodn_cloud_optimised/bin/mooring_temperature_logger_delayed_qc.py
@@ -18,7 +18,7 @@ def main():
         "mooring_temperature_logger_delayed_qc.json",
         "--clear-existing-data",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
 
     # Run the command

diff --git a/aodn_cloud_optimised/bin/radar_BonneyCoast_velocity_hourly_averaged_delayed_qc.py b/aodn_cloud_optimised/bin/radar_BonneyCoast_velocity_hourly_averaged_delayed_qc.py
@@ -14,7 +14,7 @@ def main():
         "radar_BonneyCoast_velocity_hourly_averaged_delayed_qc.json",
         "--clear-existing-data",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
 
     # Run the command

diff --git a/aodn_cloud_optimised/bin/radar_CapricornBunkerGroup_velocity_hourly_averaged_delayed_qc.py b/aodn_cloud_optimised/bin/radar_CapricornBunkerGroup_velocity_hourly_averaged_delayed_qc.py
@@ -14,7 +14,7 @@ def main():
         "radar_CapricornBunkerGroup_velocity_hourly_averaged_delayed_qc.json",
         "--clear-existing-data",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
 
     # Run the command

diff --git a/aodn_cloud_optimised/bin/radar_CapricornBunkerGroup_wave_delayed_qc.py b/aodn_cloud_optimised/bin/radar_CapricornBunkerGroup_wave_delayed_qc.py
@@ -14,7 +14,7 @@ def main():
         "radar_CapricornBunkerGroup_wave_delayed_qc.json",
         "--clear-existing-data",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
 
     # Run the command

diff --git a/aodn_cloud_optimised/bin/radar_CapricornBunkerGroup_wind_delayed_qc.py b/aodn_cloud_optimised/bin/radar_CapricornBunkerGroup_wind_delayed_qc.py
@@ -14,7 +14,7 @@ def main():
         "radar_CapricornBunkerGroup_wind_delayed_qc.json",
         "--clear-existing-data",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
 
     # Run the command

diff --git a/aodn_cloud_optimised/bin/radar_CoffsHarbour_velocity_hourly_averaged_delayed_qc.py b/aodn_cloud_optimised/bin/radar_CoffsHarbour_velocity_hourly_averaged_delayed_qc.py
@@ -14,7 +14,7 @@ def main():
         "radar_CoffsHarbour_velocity_hourly_averaged_delayed_qc.json",
         "--clear-existing-data",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
 
     # Run the command

diff --git a/aodn_cloud_optimised/bin/radar_CoffsHarbour_wave_delayed_qc.py b/aodn_cloud_optimised/bin/radar_CoffsHarbour_wave_delayed_qc.py
@@ -14,7 +14,7 @@ def main():
         "radar_CoffsHarbour_wave_delayed_qc.json",
         "--clear-existing-data",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
 
     # Run the command

diff --git a/aodn_cloud_optimised/bin/radar_CoffsHarbour_wind_delayed_qc.py b/aodn_cloud_optimised/bin/radar_CoffsHarbour_wind_delayed_qc.py
@@ -14,7 +14,7 @@ def main():
         "radar_CoffsHarbour_wind_delayed_qc.json",
         "--clear-existing-data",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
 
     # Run the command

diff --git a/aodn_cloud_optimised/bin/radar_CoralCoast_velocity_hourly_averaged_delayed_qc.py b/aodn_cloud_optimised/bin/radar_CoralCoast_velocity_hourly_averaged_delayed_qc.py
@@ -14,7 +14,7 @@ def main():
         "radar_CoralCoast_velocity_hourly_averaged_delayed_qc.json",
         "--clear-existing-data",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
 
     # Run the command

diff --git a/aodn_cloud_optimised/bin/radar_Newcastle_velocity_hourly_averaged_delayed_qc.py b/aodn_cloud_optimised/bin/radar_Newcastle_velocity_hourly_averaged_delayed_qc.py
@@ -14,7 +14,7 @@ def main():
         "radar_Newcastle_velocity_hourly_averaged_delayed_qc.json",
         "--clear-existing-data",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
 
     # Run the command

diff --git a/aodn_cloud_optimised/bin/radar_NorthWestShelf_velocity_hourly_averaged_delayed_qc.py b/aodn_cloud_optimised/bin/radar_NorthWestShelf_velocity_hourly_averaged_delayed_qc.py
@@ -14,7 +14,7 @@ def main():
         "radar_NorthWestShelf_velocity_hourly_averaged_delayed_qc.json",
         "--clear-existing-data",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
 
     # Run the command

diff --git a/aodn_cloud_optimised/bin/radar_RottnestShelf_velocity_hourly_averaged_delayed_qc.py b/aodn_cloud_optimised/bin/radar_RottnestShelf_velocity_hourly_averaged_delayed_qc.py
@@ -14,7 +14,7 @@ def main():
         "radar_RottnestShelf_velocity_hourly_averaged_delayed_qc.json",
         "--clear-existing-data",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
 
     # Run the command

diff --git a/aodn_cloud_optimised/bin/radar_RottnestShelf_wave_delayed_qc.py b/aodn_cloud_optimised/bin/radar_RottnestShelf_wave_delayed_qc.py
@@ -14,7 +14,7 @@ def main():
         "radar_RottnestShelf_wave_delayed_qc.json",
         "--clear-existing-data",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
 
     # Run the command

diff --git a/aodn_cloud_optimised/bin/radar_RottnestShelf_wind_delayed_qc.py b/aodn_cloud_optimised/bin/radar_RottnestShelf_wind_delayed_qc.py
@@ -6,15 +6,15 @@ def main():
     command = [
         "generic_cloud_optimised_creation",
         "--paths",
-        "IMOS/ACORN/gridded_1h-avg-wind-map_QC/ROT/",
-        # "--filters",
-        # "FILTER_STRING_1",
+        "IMOS/ACORN/gridded_1h-avg-wind-map_QC/ROT/2010/04/01",
+        "--filters",
+        "20100401T0",
         # "FILTER_STRING_1",
         "--dataset-config",
         "radar_RottnestShelf_wind_delayed_qc.json",
         "--clear-existing-data",
         "--cluster-mode",
-        "remote",
+        "ec2",
     ]
 
     # Run the command

diff --git a/aodn_cloud_optimised/bin/radar_SouthAustraliaGulfs_velocity_hourly_averaged_delayed_qc.py b/aodn_cloud_optimised/bin/radar_SouthAustraliaGulfs_velocity_hourly_averaged_delayed_qc.py
@@ -14,7 +14,7 @@ def main():
         "radar_SouthAustraliaGulfs_velocity_hourly_averaged_delayed_qc.json",
         "--clear-existing-data",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
 
     # Run the command

diff --git a/aodn_cloud_optimised/bin/radar_SouthAustraliaGulfs_wave_delayed_qc.py b/aodn_cloud_optimised/bin/radar_SouthAustraliaGulfs_wave_delayed_qc.py
@@ -14,7 +14,7 @@ def main():
         "radar_SouthAustraliaGulfs_wave_delayed_qc.json",
         "--clear-existing-data",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
 
     # Run the command

diff --git a/aodn_cloud_optimised/bin/radar_SouthAustraliaGulfs_wind_delayed_qc.py b/aodn_cloud_optimised/bin/radar_SouthAustraliaGulfs_wind_delayed_qc.py
@@ -14,7 +14,7 @@ def main():
         "radar_SouthAustraliaGulfs_wind_delayed_qc.json",
         "--clear-existing-data",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
 
     # Run the command

diff --git a/aodn_cloud_optimised/bin/radar_TurquoiseCoast_velocity_hourly_average_delayed_qc.py b/aodn_cloud_optimised/bin/radar_TurquoiseCoast_velocity_hourly_average_delayed_qc.py
@@ -11,7 +11,7 @@ def main():
         "radar_TurquoiseCoast_velocity_hourly_average_delayed_qc.json",
         "--clear-existing-data",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
 
     # Run the command

diff --git a/aodn_cloud_optimised/bin/receiver_animal_acoustic_tagging_delayed_qc.py b/aodn_cloud_optimised/bin/receiver_animal_acoustic_tagging_delayed_qc.py
@@ -13,7 +13,7 @@ def main():
         "receiver_animal_acoustic_tagging_delayed_qc.json",
         "--clear-existing-data",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
 
     # Run the command

diff --git a/aodn_cloud_optimised/bin/satellite_chlorophylla_carder_1day_aqua.py b/aodn_cloud_optimised/bin/satellite_chlorophylla_carder_1day_aqua.py
@@ -14,7 +14,7 @@ def main():
         "satellite_chlorophylla_carder_1day_aqua.json",
         "--clear-existing-data",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
 
     # Run the command

diff --git a/aodn_cloud_optimised/bin/satellite_chlorophylla_gsm_1day_aqua.py b/aodn_cloud_optimised/bin/satellite_chlorophylla_gsm_1day_aqua.py
@@ -13,7 +13,7 @@ def main():
         "satellite_chlorophylla_gsm_1day_aqua.json",
         "--clear-existing-data",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
 
     # Run the command

diff --git a/aodn_cloud_optimised/bin/satellite_chlorophylla_oci_1day_aqua.py b/aodn_cloud_optimised/bin/satellite_chlorophylla_oci_1day_aqua.py
@@ -9,7 +9,7 @@ def main():
     command = [
         "generic_cloud_optimised_creation",
         "--paths",
-        "IMOS/SRS/OC/gridded/aqua/P1D/2024",
+        "IMOS/SRS/OC/gridded/aqua/P1D/",
         "--filters",
         "aust.chl_oci.nc",
         "--dataset-config",

diff --git a/aodn_cloud_optimised/bin/satellite_diffuse_attenuation_coefficent_1day_aqua.py b/aodn_cloud_optimised/bin/satellite_diffuse_attenuation_coefficent_1day_aqua.py
@@ -13,7 +13,7 @@ def main():
         "satellite_diffuse_attenuation_coefficent_1day_aqua.json",
         "--clear-existing-data",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
 
     # Run the command

diff --git a/aodn_cloud_optimised/bin/satellite_ghrsst_l3c_1day_nighttime_himawari8.py b/aodn_cloud_optimised/bin/satellite_ghrsst_l3c_1day_nighttime_himawari8.py
@@ -14,7 +14,7 @@ def main():
         "satellite_ghrsst_l3c_1day_nighttime_himawari8.json",
         "--clear-existing-data",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
 
     # Run the command

diff --git a/..._cloud_optimised/bin/satellite_ghrsst_l3s_1day_daynighttime_multi_sensor_southernocean.py b/..._cloud_optimised/bin/satellite_ghrsst_l3s_1day_daynighttime_multi_sensor_southernocean.py
@@ -19,7 +19,7 @@ def main():
         "satellite_ghrsst_l3s_1day_daynighttime_multi_sensor_southernocean.json",
         "--clear-existing-data",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
 
     # Run the command

diff --git a/aodn_cloud_optimised/bin/satellite_ghrsst_l3s_1day_daynighttime_single_sensor_australia.py b/aodn_cloud_optimised/bin/satellite_ghrsst_l3s_1day_daynighttime_single_sensor_australia.py
@@ -46,7 +46,7 @@ def main():
         "satellite_ghrsst_l3s_1day_daynighttime_single_sensor_australia.json",
         "--clear-existing-data",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
 
     # Run the command