Skip to content

Commit

Permalink
Refactor config file parsing
Browse files Browse the repository at this point in the history
  • Loading branch information
@balazsdukai
balazsdukai committed Jul 23, 2019
1 parent ad9d24e commit b286b6f
Show file tree
Hide file tree
Showing 7 changed files with 179 additions and 166 deletions.
10 changes: 9 additions & 1 deletion .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,16 @@
dist
deploy.sh

# Eclipse
# Virtualenv
Pipfile.lock

# IDE
.settings
.project
.pydevproject
.dbeaver-data-sources.xml
.idea
.vscode

# SQL
Script*
Expand Down Expand Up @@ -37,3 +42,6 @@ example_data/ahn2/las
example_data/ahn3/las
*.qgz
*.qgs*

# KDE
.directory
116 changes: 58 additions & 58 deletions bag3d/app.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,20 +55,20 @@ def app(cli_args, here, log_conf):
try:
conn = db.db(
dbname=cfg['database']['dbname'],
host=str(cfg['database']["host"]),
port=str(cfg['database']["port"]),
user=cfg['database']["user"],
password=cfg['database']["pw"])
host=str(cfg['database']['host']),
port=str(cfg['database']['port']),
user=cfg['database']['user'],
password=cfg['database']['pw'])
except Exception as e:
logger.exception(e)
sys.exit(1)

try:
# well, let's assume the user provided the AHN3 dir first
ahn3_fp = cfg["input_elevation"]["dataset_name"][0]
ahn2_fp = cfg["input_elevation"]["dataset_name"][1]
ahn3_dir = cfg["input_elevation"]["dataset_dir"][0]
ahn2_dir = cfg["input_elevation"]["dataset_dir"][1]
ahn3_fp = cfg['input_elevation']['dataset_name'][0]
ahn2_fp = cfg['input_elevation']['dataset_name'][1]
ahn3_dir = cfg['input_elevation']['dataset_dir'][0]
ahn2_dir = cfg['input_elevation']['dataset_dir'][1]


if args_in['update_bag']:
Expand All @@ -84,66 +84,66 @@ def app(cli_args, here, log_conf):
ahn.download(path_lasinfo=cfg['path_lasinfo'],
ahn3_dir=ahn3_dir,
ahn2_dir=ahn2_dir,
tile_index_file=cfg["elevation"]["file"],
tile_index_file=cfg['tile_index']['elevation']['file'],
ahn3_file_pat=ahn3_fp,
ahn2_file_pat=ahn2_fp)


if args_in['update_ahn_raster']:
logger.info("Updating AHN 0.5m raster files")
ahn.download_raster(conn, cfg,
cfg['quality']["ahn2_rast_dir"],
cfg['quality']["ahn3_rast_dir"],
cfg['quality']['ahn2_rast_dir'],
cfg['quality']['ahn3_rast_dir'],
doexec=args_in['no_exec'])

if args_in['import_tile_idx']:
logger.info("Importing BAG tile index")
bag.import_index(cfg['polygons']["file"], cfg['database']['dbname'],
cfg['polygons']['schema'], str(cfg['database']["host"]),
str(cfg['database']["port"]), cfg['database']["user"],
cfg['database']["pw"],
bag.import_index(cfg['tile_index']['polygons']['file'], cfg['database']['dbname'],
cfg['tile_index']['polygons']['schema'], str(cfg['database']['host']),
str(cfg['database']['port']), cfg['database']['user'],
cfg['database']['pw'],
doexec=args_in['no_exec'])
# Update BAG tiles to include the lower/left boundary
footprints.update_tile_index(conn,
table_index=[cfg['polygons']['schema'],
cfg['polygons']["table"]],
fields_index=[cfg['polygons']['fields']["primary_key"],
cfg['polygons']['fields']["geometry"],
cfg['polygons']['fields']["unit_name"]]
table_index=[cfg['tile_index']['polygons']['schema'],
cfg['tile_index']['polygons']['table']],
fields_index=[cfg['tile_index']['polygons']['fields']['primary_key'],
cfg['tile_index']['polygons']['fields']['geometry'],
cfg['tile_index']['polygons']['fields']['unit_name']]
)
logger.info("Partitioning the BAG")
logger.debug("Creating centroids")
footprints.create_centroids(conn,
table_centroid=[cfg['footprints']['schema'],
"pand_centroid"],
table_footprint=[cfg['footprints']['schema'],
cfg['footprints']["table"]],
fields_footprint=[cfg['footprints']['fields']["primary_key"],
cfg['footprints']['fields']["geometry"]]
table_centroid=[cfg['input_polygons']['footprints']['schema'],
'pand_centroid'],
table_footprint=[cfg['input_polygons']['footprints']['schema'],
cfg['input_polygons']['footprints']['table']],
fields_footprint=[cfg['input_polygons']['footprints']['fields']['primary_key'],
cfg['input_polygons']['footprints']['fields']['geometry']]
)
logger.debug("Creating tiles")
footprints.create_views(conn, schema_tiles=cfg['tile_schema'],
table_index=[cfg['polygons']['schema'],
cfg['polygons']["table"]],
fields_index=[cfg['polygons']['fields']["primary_key"],
cfg['polygons']['fields']["geometry"],
cfg['polygons']['fields']["unit_name"]],
table_centroid=[cfg['footprints']['schema'], "pand_centroid"],
fields_centroid=[cfg['footprints']['fields']["primary_key"],
"geom"],
table_footprint=[cfg['footprints']['schema'],
cfg['footprints']["table"]],
fields_footprint=[cfg['footprints']['fields']["primary_key"],
cfg['footprints']['fields']["geometry"],
cfg['footprints']['fields']["uniqueid"]
footprints.create_views(conn, schema_tiles=cfg['input_polygons']['tile_schema'],
table_index=[cfg['tile_index']['polygons']['schema'],
cfg['tile_index']['polygons']['table']],
fields_index=[cfg['tile_index']['polygons']['fields']['primary_key'],
cfg['tile_index']['polygons']['fields']['geometry'],
cfg['tile_index']['polygons']['fields']['unit_name']],
table_centroid=[cfg['input_polygons']['footprints']['schema'], 'pand_centroid'],
fields_centroid=[cfg['input_polygons']['footprints']['fields']['primary_key'],
'geom'],
table_footprint=[cfg['input_polygons']['footprints']['schema'],
cfg['input_polygons']['footprints']['table']],
fields_footprint=[cfg['input_polygons']['footprints']['fields']['primary_key'],
cfg['input_polygons']['footprints']['fields']['geometry'],
cfg['input_polygons']['footprints']['fields']['uniqueid']
],
prefix_tiles=cfg['prefix_tile_footprint'])
prefix_tiles=cfg['input_polygons']['tile_prefix'])

logger.info("Importing AHN tile index")
bag.import_index(cfg['elevation']['file'], cfg['database']['dbname'],
cfg['elevation']['schema'], str(cfg['database']["host"]),
str(cfg['database']["port"]), cfg['database']["user"],
cfg['database']["pw"],
bag.import_index(cfg['tile_index']['elevation']['file'], cfg['database']['dbname'],
cfg['tile_index']['elevation']['schema'], str(cfg['database']['host']),
str(cfg['database']['port']), cfg['database']['user'],
cfg['database']['pw'],
doexec=args_in['no_exec'])


Expand Down Expand Up @@ -192,7 +192,7 @@ def app(cli_args, here, log_conf):

if not os.listdir(c['output']['staging']['dir']):
logger.warning("3dfier failed completely for %s, skipping import",
c["config"]["in"])
c['config']['in'])
else:
logger.info("Importing batch3dfier output into database")
importer.import_csv(conn, c)
Expand All @@ -214,14 +214,14 @@ def app(cli_args, here, log_conf):
logger.info("Migrating the 3D BAG to production")
exporter.migrate(conn, cfg)
logger.info("Exporting 3D BAG")
exporter.csv(conn, cfg, cfg['output']["production"]["dir"])
exporter.gpkg(conn, cfg, cfg['output']["production"]["dir"], args_in['no_exec'])
exporter.postgis(conn, cfg, cfg['output']["production"]["dir"], args_in['no_exec'])
exporter.csv(conn, cfg, cfg['output']['production']['dir'])
exporter.gpkg(conn, cfg, cfg['output']['production']['dir'], args_in['no_exec'])
exporter.postgis(conn, cfg, cfg['output']['production']['dir'], args_in['no_exec'])

if args_in['grant_access']:
bag.grant_access(conn, args_in["grant_access"],
cfg['tile_schema'],
cfg['polygons']['schema'],
bag.grant_access(conn, args_in['grant_access'],
cfg['input_polygons']['tile_schema'],
cfg['tile_index']['polygons']['schema'],
cfg['output']['production']['schema'])

if args_in['quality']:
Expand All @@ -240,11 +240,11 @@ def app(cli_args, here, log_conf):
cfg['config']['out_border_ahn3'],
cfg['config']['out_rest']]:
rmtree(os.path.dirname(c), ignore_errors=True)
# rmtree(c['output']["dir"], ignore_errors=True)
# rmtree(c['output']['dir'], ignore_errors=True)

# rast_idx = ahn.rast_file_idx(conn, cfg,
# cfg['quality']["ahn2_rast_dir"],
# cfg['quality']["ahn3_rast_dir"])
# cfg['quality']['ahn2_rast_dir'],
# cfg['quality']['ahn3_rast_dir'])
# sample = quality.get_sample(conn, cfg_quality)
# logger.info("Sample size %s", len(sample))
# logger.debug(sample[0])
Expand All @@ -255,11 +255,11 @@ def app(cli_args, here, log_conf):
# diffs,fields = quality.compute_diffs(reference, stats)
# logger.info("Computed differences on %s buildings", len(diffs))
#
# out_dir = os.path.dirname(cfg['quality']["results"])
# out_dir = os.path.dirname(cfg['quality']['results'])
# os.makedirs(out_dir, exist_ok=True)
# logger.info("Writing height comparison to %s",
# cfg['quality']["results"])
# with open(cfg['quality']["results"], 'w') as csvfile:
# cfg['quality']['results'])
# with open(cfg['quality']['results'], 'w') as csvfile:
# writer = DictWriter(csvfile, fieldnames=fields)
# writer.writeheader()
# for row in diffs:
Expand Down
44 changes: 22 additions & 22 deletions bag3d/batch3dfier/process.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,16 +23,16 @@ def run(conn, config, doexec=True):
list of str
The tiles that failed
"""
if config["input_polygons"]["tile_list"] is None:
logger.error("tile_list in %s is empty, exiting function", config["config"]["in"])
if config['input_polygons']['tile_list'] is None:
logger.error("tile_list in %s is empty, exiting function", config['config']['in'])
return None
else:
tiles = config["input_polygons"]["tile_list"]
cfg_dir = os.path.dirname(config["config"]["in"])
pc_name_map = batch3dfier.pc_name_dict(config["input_elevation"]["dataset_dir"],
config["input_elevation"]["dataset_name"])
tiles = config['input_polygons']['tile_list']
cfg_dir = os.path.dirname(config['config']['in'])
pc_name_map = batch3dfier.pc_name_dict(config['input_elevation']['dataset_dir'],
config['input_elevation']['dataset_name'])
pc_file_idx = batch3dfier.pc_file_index(pc_name_map)
tile_group = re.search(r"bag3d_cfg_(\w+).yml", config["config"]["in"]).group(1)
tile_group = re.search(r"bag3d_cfg_(\w+).yml", config['config']['in']).group(1)

exitFlag = 0
tiles_skipped = []
Expand All @@ -59,18 +59,18 @@ def process_data(threadName, q):
t = batch3dfier.call_3dfier(
db=conn,
tile=tile,
schema_tiles=config["input_polygons"]['user_schema'],
table_index_pc=config["tile_index"]['elevation'],
fields_index_pc=config["tile_index"]['elevation']['fields'],
idx_identical=config["tile_index"]["identical"],
table_index_footprint=config["tile_index"]['polygons'],
fields_index_footprint=config["tile_index"]['polygons']['fields'],
uniqueid=config["input_polygons"]["footprints"]["fields"]['uniqueid'],
extent_ewkb=config["extent_ewkb"],
clip_prefix=config["clip_prefix"],
prefix_tile_footprint=config["input_polygons"]['tile_prefix'],
schema_tiles=config['input_polygons']['user_schema'],
table_index_pc=config['tile_index']['elevation'],
fields_index_pc=config['tile_index']['elevation']['fields'],
idx_identical=config['tile_index']['identical'],
table_index_footprint=config['tile_index']['polygons'],
fields_index_footprint=config['tile_index']['polygons']['fields'],
uniqueid=config['input_polygons']['footprints']['fields']['uniqueid'],
extent_ewkb=config['extent_ewkb'],
clip_prefix=config['clip_prefix'],
prefix_tile_footprint=config['input_polygons']['tile_prefix'],
yml_dir=cfg_dir,
tile_out=config["tile_out"],
tile_out=config['tile_out'],
output_format='CSV-BUILDINGS-MULTIPLE',
output_dir=config['output']['staging']['dir'],
path_3dfier=config['path_3dfier'],
Expand All @@ -86,7 +86,7 @@ def process_data(threadName, q):
queueLock.release()

# Prep
threadList = ["Thread-" + str(t + 1) for t in range(config['threads'])]
threadList = ["Thread-" + str(t + 1) for t in range(config['config']['threads'])]
queueLock = threading.Lock()
workQueue = queue.Queue(0)
threads = []
Expand Down Expand Up @@ -114,12 +114,12 @@ def process_data(threadName, q):
# Drop temporary views that reference the clipped extent
try:
to_drop = [tile for tile in tiles if
config["clip_prefix"] in tile or
config["tile_out"] in tile]
config['clip_prefix'] in tile or
config['tile_out'] in tile]
if to_drop:
batch3dfier.drop_2Dtiles(
conn,
config["input_polygons"]['user_schema'],
config['input_polygons']['user_schema'],
views_to_drop=to_drop)
except TypeError:
logger.debug("No views to drop")
Expand Down
Loading

0 comments on commit b286b6f

Please sign in to comment.