Skip to content

Commit

Permalink
new file for single year update
Browse files Browse the repository at this point in the history
  • Loading branch information
@giosans
giosans committed Mar 27, 2023
1 parent 2fe6e10 commit 9246f36
Show file tree
Hide file tree
Showing 2 changed files with 369 additions and 2 deletions.
3 changes: 1 addition & 2 deletions notebooks/Vaklodingen2EE.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@
"metadata": {},
"outputs": [],
"source": [
"#url = 'http://opendap.deltares.nl/thredds/catalog/opendap/rijkswaterstaat/kusthoogte/catalog.html'\n",
"url_catalog = 'https://opendap.deltares.nl/thredds/catalog/opendap/rijkswaterstaat/vaklodingen_new/catalog.html'\n",
"url_base = 'http://opendap.deltares.nl/thredds/dodsC/opendap/rijkswaterstaat/vaklodingen_new'\n",
"ext = 'nc'\n",
Expand Down Expand Up @@ -329,7 +328,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.12"
"version": "3.9.16"
}
},
"nbformat": 4,
Expand Down
368 changes: 368 additions & 0 deletions notebooks/Vaklodingen2EE_SingleYearUpdate.ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,368 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#%matplotlib inline\n",
"\n",
"import os\n",
"import subprocess\n",
"import itertools\n",
"import numpy as np\n",
"import requests\n",
"import pytz\n",
"import datetime\n",
"import netCDF4\n",
"from osgeo import gdal\n",
"from os import path\n",
"from osgeo.gdalconst import *\n",
"from tqdm import tqdm\n",
"from bs4 import BeautifulSoup\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"url_catalog = 'https://opendap.deltares.nl/thredds/catalog/opendap/rijkswaterstaat/vaklodingen_new/catalog.html'\n",
"url_base = 'http://opendap.deltares.nl/thredds/dodsC/opendap/rijkswaterstaat/vaklodingen_new'\n",
"ext = 'nc'\n",
"urls = []\n",
"yearupdate = 2022 # change year for update\n",
"\n",
"def listFD(url, ext=''):\n",
" page = requests.get(url).text\n",
" soup = BeautifulSoup(page, 'html.parser')\n",
"\n",
" return [url + '/' + node.get('href') for node in soup.find_all('a') if node.get('href').endswith(ext)]\n",
"\n",
"\n",
"for ncfile in listFD(url_catalog, ext):\n",
" items = ncfile.split('/catalog.html/')\n",
" filename = items[1].split('/')[-1]\n",
" url = url_base + '/' + filename\n",
" if filename == 'catalog.nc':\n",
" continue\n",
" urls.append(url)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"urls[:]\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"grids = []\n",
"for url in tqdm(urls[:]):\n",
" ds = netCDF4.Dataset(url)\n",
" times = netCDF4.num2date(ds.variables['time'][:], ds.variables['time'].units, calendar='julian')\n",
" idbooltime = [1 if t.year == yearupdate else 0 for t in times ]\n",
" idtime = np.where(idbooltime)[0]\n",
" local = pytz.timezone(\"Europe/Amsterdam\")\n",
" times = [datetime.datetime.strptime(t.isoformat(), \"%Y-%m-%dT%H:%M:%S\").replace(tzinfo=pytz.utc) \n",
" for t in times if t.year == yearupdate]\n",
" if len(times) == 0:\n",
" continue\n",
" \n",
" arrs = []\n",
" z = ds.variables['z'][idtime,:,:]\n",
" x = ds.variables['x'][:]\n",
" y = ds.variables['y'][:]\n",
"\n",
" grids.append({\n",
" \"url\": url,\n",
" \"x\": x,\n",
" \"y\": y,\n",
" \"z\": z,\n",
" \"times\": times\n",
" })\n",
" ds.close()\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"count = len(list(itertools.chain.from_iterable([g['times'] for g in grids])))\n",
"count"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"print(grids[0]['z'][0])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"\n",
"#cmd\n",
"#subprocess.call('gsutil cp '../output/bathymetry_1985_0001.tif' gs://eo-bathymetry-rws/vaklodingen/bathymetry_1985_0001.tif', shell=True)\n",
"#ccc=r\"dir\"\n",
"#ccc\n",
"#subprocess.call(ccc)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Make sure you create the image collection folder in google earth engine before running"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"ee_collection_path = 'projects/deltares-rws/eo-bathymetry/vaklodingen'"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"def run(cmd, shell=True):\n",
" # print(cmd)\n",
" subprocess.call(cmd,shell=shell)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#for g in tqdm(grids):\n",
"# print(g['times'])"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████████████████████████████████████████████████████████████████████| 72/72 [02:37<00:00, 2.19s/it]\n"
]
}
],
"source": [
"start_index = 0\n",
"dirbathy = r'../../output_vaklodingen/'\n",
"j = 0\n",
"ts = []\n",
"if not os.path.exists(dirbathy):\n",
" os.makedirs(dirbathy)\n",
"for g in tqdm(grids):\n",
" ncols = len(g['x'])\n",
" nrows = len(g['y'])\n",
" cellsize = g['x'][1] - g['x'][0]\n",
" # taking corners\n",
" xllcorner = np.min(g['x']-10)\n",
" yllcorner = np.min(g['y']-10)\n",
" nodata_value = -32767\n",
" z = g['z']\n",
" #print(z.shape)\n",
"\n",
" for i, t in enumerate(g['times']):\n",
" ts.append(t)\n",
" if i < start_index:\n",
" i = i + 1\n",
" continue\n",
" j += 1\n",
" \n",
" filename = 'vaklodingen_' + str(str(t)[:4]) + '_' + str(j).rjust(4, '0')\n",
" filepath = dirbathy + filename\n",
" filepath_asc = filepath + '.asc'\n",
" filepath_tif = filepath + '.tif'\n",
"\n",
" zi = z[i]\n",
"\n",
" with open(filepath_asc, 'w') as f:\n",
" f.write('ncols {0}\\n'.format(ncols))\n",
" f.write('nrows {0}\\n'.format(nrows))\n",
" f.write('cellsize {0}\\n'.format(cellsize))\n",
" f.write('xllcorner {0}\\n'.format(xllcorner))\n",
" f.write('yllcorner {0}\\n'.format(yllcorner))\n",
" f.write('nodata_value {0}\\n'.format(nodata_value))\n",
" for row in range(nrows-1,-1,-1):\n",
" s = ' '.join([str(v) for v in zi[row,]]).replace('--', str(nodata_value))\n",
" f.write(s)\n",
" f.write('\\n')\n",
"\n",
" #cmd = 'gdal_translate -ot Float32 -a_srs EPSG:28992 -co COMPRESS=DEFLATE -co PREDICTOR=2 -co ZLEVEL=6 -of GTiff {0} {1}'\\\n",
" # .format(filepath_asc, filepath_tif)\n",
" # per tile\n",
" cmd = 'gdal_translate -ot Float32 -a_srs EPSG:28992 -of COG {0} {1}'\\\n",
" .format(filepath_asc, filepath_tif)\n",
" run(cmd)\n"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
"nodata_value = -32767"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"1it [03:34, 214.61s/it]\n"
]
}
],
"source": [
"# merge per year\n",
"tzinfo = ts[0].tzinfo\n",
"uyears = list(dict.fromkeys(map(lambda x: x.year, ts))) # unique years\n",
"uts = list(map(lambda x: datetime.datetime(year=x, month=1, day=1).replace(tzinfo=tzinfo), uyears)) # unique times\n",
"\n",
"for ii, tt in tqdm(enumerate(uyears)):\n",
" filename = 'vaklodingen_' + str(str(tt)[:4])\n",
" filepath = dirbathy + filename\n",
" filepath_tif = [dirbathy+ll for ll in os.listdir(dirbathy) if str(tt) in ll.split('_')[1] and ll.endswith('.tif')]\n",
" filepath_year_tif = filepath + '.tif'\n",
" \n",
" # per year\n",
" files_to_mosaic = filepath_tif \n",
" g = gdal.Warp(filepath_year_tif, files_to_mosaic, dstSRS='EPSG:28992', \n",
" outputType=gdal.GDT_Float32, format=\"COG\",\n",
" options=[\"COMPRESS=LZW\", \"TILED=YES\"])\n",
" g = None \n",
" \n",
" filepath_gs = 'gs://eo-bathymetry-rws/vaklodingen/' + filename # temporary file system in storage bucket\n",
" #print(filepath_gs)\n",
" cmd = 'gsutil cp {0} {1}' \\\n",
" .format(filepath_year_tif, filepath_gs)\n",
" run(cmd, shell=True)\n",
"\n",
" filepath_ee = ee_collection_path + '/' + filename\n",
" #print(filepath_ee)\n",
" cmd = 'earthengine upload image --wait --asset_id={0} --nodata_value={1} {2}' \\\n",
" .format(filepath_ee, nodata_value, filepath_gs)\n",
" run(cmd, shell=True)\n",
"\n",
" time_start = int(uts[ii].timestamp() * 1000)\n",
" cmd = 'earthengine asset set --time_start {0} {1}' \\\n",
" .format(time_start, filepath_ee)\n",
" run(cmd, shell=True)\n",
"\n",
" cmd = 'earthengine acl set public {0}' \\\n",
" .format(filepath_ee)\n",
" run(cmd, shell=True)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# following is just for testing."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
" filepath_gs = 'gs://eo-bathymetry-rws/jarkus/' + filename_tif\n",
" \n",
" #gsutil = 'D:/src/google-cloud-sdk/bin/gsutil.cmd' # relative path is not defined on Windows\n",
" gsutil = 'gsutil'\n",
" cmd = gsutil + ' cp {0} {1}'\\\n",
" .format(filepath_tif, filepath_gs)\n",
" run(cmd)\n",
" \n",
" filepath_ee = ee_collection_path + '/' + filename \n",
" cmd = 'earthengine upload image --wait --asset_id={0} --nodata_value={1} {2}'\\\n",
" .format(filepath_ee, nodata_value, filepath_gs) \n",
" run(cmd)\n",
" \n",
" time_start = int(grids[0]['times'][0].timestamp() * 1000)\n",
" cmd = 'earthengine asset set --time_start {0} {1}'\\\n",
" .format(time_start, filepath_ee)\n",
" run(cmd)\n",
"\n",
" cmd = 'earthengine acl set public {0}'\\\n",
" .format(filepath_ee)\n",
" run(cmd)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"anaconda-cloud": {},
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.16"
}
},
"nbformat": 4,
"nbformat_minor": 2
}

0 comments on commit 9246f36

Please sign in to comment.