Merge pull request #101 from aodn/SrsOcNew2

Feat: add new SRS OC collections
aodn · Nov 21, 2024 · 078c14b · 078c14b
2 parents d6102f0 + 00a9c58
commit 078c14b
Show file tree

Hide file tree

Showing 169 changed files with 58,066 additions and 636 deletions.
diff --git a/aodn_cloud_optimised/bin/argo.py b/aodn_cloud_optimised/bin/argo.py
@@ -21,7 +21,7 @@ def main():
     #     # "--clear-existing-data",
     #     "--force-previous-parquet-deletion",
     #     "--cluster-mode",
-    #     "remote",
+    #     "coiled",
     # ]
     #
     # # Run the command
@@ -38,7 +38,7 @@ def main():
         "argo.json",
         "--force-previous-parquet-deletion",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
     subprocess.run(command, check=True)
 
@@ -56,7 +56,7 @@ def main():
         "argo.json",
         "--force-previous-parquet-deletion",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
     subprocess.run(command, check=True)
 
@@ -73,6 +73,6 @@ def main():
         "argo.json",
         "--force-previous-parquet-deletion",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
     subprocess.run(command, check=True)
diff --git a/aodn_cloud_optimised/bin/autonomous_underwater_vehicle.py b/aodn_cloud_optimised/bin/autonomous_underwater_vehicle.py
@@ -895,7 +895,7 @@ def main():
         "autonomous_underwater_vehicle.json",
         "--clear-existing-data",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
 
     # Run the command

diff --git a/aodn_cloud_optimised/bin/create_dataset_config.py b/aodn_cloud_optimised/bin/create_dataset_config.py
@@ -204,7 +204,7 @@ def main():
         "{dataset_json}",
         "--clear-existing-data",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
 
     # Run the command
@@ -678,7 +678,7 @@ def main():
     dataset_config["metadata_uuid"] = args.uuid
     dataset_config["logger_name"] = args.dataset_name
     dataset_config["cloud_optimised_format"] = args.cloud_format
-    dataset_config["cluster_options"] = {
+    dataset_config["coiled_cluster_options"] = {
         "n_workers": [1, 20],
         "scheduler_vm_types": "t3.small",
         "worker_vm_types": "t3.medium",

diff --git a/aodn_cloud_optimised/bin/generic_cloud_optimised_creation.py b/aodn_cloud_optimised/bin/generic_cloud_optimised_creation.py
@@ -9,13 +9,13 @@
 Usage Examples:
   generic_cloud_optimised_creation --paths 'IMOS/ANMN/NSW' 'IMOS/ANMN/PA'
   --filters '_hourly-timeseries_' 'FV02' --dataset-config 'mooring_hourly_timeseries_delayed_qc.json'
-  --clear-existing-data --cluster-mode 'remote'
+  --clear-existing-data --cluster-mode 'coiled'
 
   generic_cloud_optimised_creation --paths 'IMOS/ANMN/NSW' 'IMOS/ANMN/QLD'
   --dataset-config 'mooring_ctd_delayed_qc.json'
 
   generic_cloud_optimised_creation --paths 'IMOS/ACORN/gridded_1h-avg-current-map_QC/TURQ/2024'
-  --dataset-config 'radar_TurquoiseCoast_velocity_hourly_average_delayed_qc.json' --clear-existing-data --cluster-mode 'remote'
+  --dataset-config 'radar_TurquoiseCoast_velocity_hourly_average_delayed_qc.json' --clear-existing-data --cluster-mode 'coiled'
 
 Arguments:
   --paths: List of S3 paths to process. Example: 'IMOS/ANMN/NSW' 'IMOS/ANMN/PA'
@@ -25,7 +25,7 @@
   --clear-existing-data: Flag to clear existing data. Default is False.
   --force-previous-parquet-deletion: Flag to force the search of previous equivalent parquet file created. Much slower.
                                      Default is False. Only for Parquet processing.
-  --cluster-mode: Cluster mode to use. Options: 'local' or 'remote'. Default is 'local'.
+  --cluster-mode: Cluster mode to use. Options: 'local' or 'coiled'. Default is 'local'.
   --optimised-bucket-name: Bucket name where cloud optimised object will be created. Default is the value of
                            BUCKET_OPTIMISED_DEFAULT from the config.
   --root-prefix-cloud-optimised-path: Prefix value for the root location of the cloud optimised objects. Default is the
@@ -51,9 +51,9 @@ def main():
     parser = argparse.ArgumentParser(
         description="Process S3 paths and create cloud-optimized datasets.",
         epilog="Examples:\n"
-        "  generic_cloud_optimised_creation --paths 'IMOS/ANMN/NSW' 'IMOS/ANMN/PA' --filters '_hourly-timeseries_' 'FV02' --dataset-config 'mooring_hourly_timeseries_delayed_qc.json' --clear-existing-data --cluster-mode 'remote'\n"
+        "  generic_cloud_optimised_creation --paths 'IMOS/ANMN/NSW' 'IMOS/ANMN/PA' --filters '_hourly-timeseries_' 'FV02' --dataset-config 'mooring_hourly_timeseries_delayed_qc.json' --clear-existing-data --cluster-mode 'coiled'\n"
         "  generic_cloud_optimised_creation --paths 'IMOS/ANMN/NSW' 'IMOS/ANMN/QLD' --dataset-config 'mooring_ctd_delayed_qc.json'\n"
-        "  generic_cloud_optimised_creation --paths 'IMOS/ACORN/gridded_1h-avg-current-map_QC/TURQ/2024' --dataset-config 'radar_TurquoiseCoast_velocity_hourly_average_delayed_qc.json' --clear-existing-data --cluster-mode 'remote'\n",
+        "  generic_cloud_optimised_creation --paths 'IMOS/ACORN/gridded_1h-avg-current-map_QC/TURQ/2024' --dataset-config 'radar_TurquoiseCoast_velocity_hourly_average_delayed_qc.json' --clear-existing-data --cluster-mode 'coiled'\n",
         formatter_class=argparse.RawTextHelpFormatter,
     )
 
@@ -91,13 +91,13 @@ def main():
         "Only for Parquet processing.",
     )
 
-    cluster_options = [mode.value for mode in clusterLib.ClusterMode]
+    coiled_cluster_options = [mode.value for mode in clusterLib.ClusterMode]
     parser.add_argument(
         "--cluster-mode",
         # type=clusterLib.parse_cluster_mode,
         default=clusterLib.ClusterMode.NONE.value,
-        choices=cluster_options,
-        help="Cluster mode to use. Options: 'local' or 'remote'. Default is None.",
+        choices=coiled_cluster_options,
+        help="Cluster mode to use. Options: 'local' or 'coiled'. Default is None.",
     )
 
     parser.add_argument(

diff --git a/aodn_cloud_optimised/bin/model_sea_level_anomaly_gridded_realtime.py b/aodn_cloud_optimised/bin/model_sea_level_anomaly_gridded_realtime.py
@@ -11,7 +11,7 @@ def main():
         "model_sea_level_anomaly_gridded_realtime.json",
         "--clear-existing-data",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
 
     # Run the command

diff --git a/aodn_cloud_optimised/bin/mooring_ctd_delayed_qc.py b/aodn_cloud_optimised/bin/mooring_ctd_delayed_qc.py
@@ -18,7 +18,7 @@ def main():
         "mooring_ctd_delayed_qc.json",
         "--clear-existing-data",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
 
     # Run the command

diff --git a/aodn_cloud_optimised/bin/mooring_hourly_timeseries_delayed_qc.py b/aodn_cloud_optimised/bin/mooring_hourly_timeseries_delayed_qc.py
@@ -18,7 +18,7 @@ def main():
         "mooring_hourly_timeseries_delayed_qc.json",
         "--clear-existing-data",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
 
     # Run the command

diff --git a/aodn_cloud_optimised/bin/mooring_satellite_altimetry_calibration_validation.py b/aodn_cloud_optimised/bin/mooring_satellite_altimetry_calibration_validation.py
@@ -14,7 +14,7 @@ def main():
         "mooring_satellite_altimetry_calibration_validation.json",
         "--clear-existing-data",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
 
     # Run the command

diff --git a/aodn_cloud_optimised/bin/mooring_temperature_logger_delayed_qc.py b/aodn_cloud_optimised/bin/mooring_temperature_logger_delayed_qc.py
@@ -18,7 +18,7 @@ def main():
         "mooring_temperature_logger_delayed_qc.json",
         "--clear-existing-data",
         "--cluster-mode",
-        "remote",
+        "coiled",
     ]
 
     # Run the command

diff --git a/aodn_cloud_optimised/bin/radar_BonneyCoast_velocity_hourly_averaged_delayed_qc.py b/aodn_cloud_optimised/bin/radar_BonneyCoast_velocity_hourly_averaged_delayed_qc.py
@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+import subprocess
+
+
+def main():
+    command = [
+        "generic_cloud_optimised_creation",
+        "--paths",
+        "IMOS/ACORN/gridded_1h-avg-current-map_QC/BONC/",
+        # "--filters",
+        # "FILTER_STRING_1",
+        # "FILTER_STRING_1",
+        "--dataset-config",
+        "radar_BonneyCoast_velocity_hourly_averaged_delayed_qc.json",
+        "--clear-existing-data",
+        "--cluster-mode",
+        "coiled",
+    ]
+
+    # Run the command
+    subprocess.run(command, check=True)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/aodn_cloud_optimised/bin/radar_CapricornBunkerGroup_velocity_hourly_averaged_delayed_qc.py b/aodn_cloud_optimised/bin/radar_CapricornBunkerGroup_velocity_hourly_averaged_delayed_qc.py
@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+import subprocess
+
+
+def main():
+    command = [
+        "generic_cloud_optimised_creation",
+        "--paths",
+        "IMOS/ACORN/gridded_1h-avg-current-map_QC/CBG/",
+        # "--filters",
+        # "FILTER_STRING_1",
+        # "FILTER_STRING_1",
+        "--dataset-config",
+        "radar_CapricornBunkerGroup_velocity_hourly_averaged_delayed_qc.json",
+        "--clear-existing-data",
+        "--cluster-mode",
+        "coiled",
+    ]
+
+    # Run the command
+    subprocess.run(command, check=True)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/aodn_cloud_optimised/bin/radar_CapricornBunkerGroup_wave_delayed_qc.py b/aodn_cloud_optimised/bin/radar_CapricornBunkerGroup_wave_delayed_qc.py
@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+import subprocess
+
+
+def main():
+    command = [
+        "generic_cloud_optimised_creation",
+        "--paths",
+        "IMOS/ACORN/gridded_1h-avg-wave-site-map_QC/CBG/",
+        # "--filters",
+        # "FILTER_STRING_1",
+        # "FILTER_STRING_1",
+        "--dataset-config",
+        "radar_CapricornBunkerGroup_wave_delayed_qc.json",
+        "--clear-existing-data",
+        "--cluster-mode",
+        "coiled",
+    ]
+
+    # Run the command
+    subprocess.run(command, check=True)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/aodn_cloud_optimised/bin/radar_CapricornBunkerGroup_wind_delayed_qc.py b/aodn_cloud_optimised/bin/radar_CapricornBunkerGroup_wind_delayed_qc.py
@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+import subprocess
+
+
+def main():
+    command = [
+        "generic_cloud_optimised_creation",
+        "--paths",
+        "IMOS/ACORN/gridded_1h-avg-wind-map_QC/CBG/",
+        # "--filters",
+        # "FILTER_STRING_1",
+        # "FILTER_STRING_1",
+        "--dataset-config",
+        "radar_CapricornBunkerGroup_wind_delayed_qc.json",
+        "--clear-existing-data",
+        "--cluster-mode",
+        "coiled",
+    ]
+
+    # Run the command
+    subprocess.run(command, check=True)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/aodn_cloud_optimised/bin/radar_CoffsHarbour_velocity_hourly_averaged_delayed_qc.py b/aodn_cloud_optimised/bin/radar_CoffsHarbour_velocity_hourly_averaged_delayed_qc.py
@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+import subprocess
+
+
+def main():
+    command = [
+        "generic_cloud_optimised_creation",
+        "--paths",
+        "IMOS/ACORN/gridded_1h-avg-current-map_QC/COF/",
+        # "--filters",
+        # "FILTER_STRING_1",
+        # "FILTER_STRING_1",
+        "--dataset-config",
+        "radar_CoffsHarbour_velocity_hourly_averaged_delayed_qc.json",
+        "--clear-existing-data",
+        "--cluster-mode",
+        "coiled",
+    ]
+
+    # Run the command
+    subprocess.run(command, check=True)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/aodn_cloud_optimised/bin/radar_CoffsHarbour_wave_delayed_qc.py b/aodn_cloud_optimised/bin/radar_CoffsHarbour_wave_delayed_qc.py
@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+import subprocess
+
+
+def main():
+    command = [
+        "generic_cloud_optimised_creation",
+        "--paths",
+        "IMOS/ACORN/gridded_1h-avg-wave-site-map_QC/COF/",
+        # "--filters",
+        # "FILTER_STRING_1",
+        # "FILTER_STRING_1",
+        "--dataset-config",
+        "radar_CoffsHarbour_wave_delayed_qc.json",
+        "--clear-existing-data",
+        "--cluster-mode",
+        "coiled",
+    ]
+
+    # Run the command
+    subprocess.run(command, check=True)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/aodn_cloud_optimised/bin/radar_CoffsHarbour_wind_delayed_qc.py b/aodn_cloud_optimised/bin/radar_CoffsHarbour_wind_delayed_qc.py
@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+import subprocess
+
+
+def main():
+    command = [
+        "generic_cloud_optimised_creation",
+        "--paths",
+        "IMOS/ACORN/gridded_1h-avg-wind-map_QC/COF/",
+        # "--filters",
+        # "FILTER_STRING_1",
+        # "FILTER_STRING_1",
+        "--dataset-config",
+        "radar_CoffsHarbour_wind_delayed_qc.json",
+        "--clear-existing-data",
+        "--cluster-mode",
+        "coiled",
+    ]
+
+    # Run the command
+    subprocess.run(command, check=True)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/aodn_cloud_optimised/bin/radar_CoralCoast_velocity_hourly_averaged_delayed_qc.py b/aodn_cloud_optimised/bin/radar_CoralCoast_velocity_hourly_averaged_delayed_qc.py
@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+import subprocess
+
+
+def main():
+    command = [
+        "generic_cloud_optimised_creation",
+        "--paths",
+        "IMOS/ACORN/gridded_1h-avg-current-map_QC/CORL/",
+        # "--filters",
+        # "FILTER_STRING_1",
+        # "FILTER_STRING_1",
+        "--dataset-config",
+        "radar_CoralCoast_velocity_hourly_averaged_delayed_qc.json",
+        "--clear-existing-data",
+        "--cluster-mode",
+        "coiled",
+    ]
+
+    # Run the command
+    subprocess.run(command, check=True)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/aodn_cloud_optimised/bin/radar_Newcastle_velocity_hourly_averaged_delayed_qc.py b/aodn_cloud_optimised/bin/radar_Newcastle_velocity_hourly_averaged_delayed_qc.py
@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+import subprocess
+
+
+def main():
+    command = [
+        "generic_cloud_optimised_creation",
+        "--paths",
+        "IMOS/ACORN/gridded_1h-avg-current-map_QC/NEWC/",
+        # "--filters",
+        # "FILTER_STRING_1",
+        # "FILTER_STRING_1",
+        "--dataset-config",
+        "radar_Newcastle_velocity_hourly_averaged_delayed_qc.json",
+        "--clear-existing-data",
+        "--cluster-mode",
+        "coiled",
+    ]
+
+    # Run the command
+    subprocess.run(command, check=True)
+
+
+if __name__ == "__main__":
+    main()