Remove parts of scripts that generate geo data and city/country trans…

…lations
todokku · Feb 11, 2020 · 2ced1ca · 2ced1ca
1 parent 2838338
commit 2ced1ca
Show file tree

Hide file tree

Showing 6 changed files with 1 addition and 268 deletions.
diff --git a/.gitignore b/.gitignore
@@ -4,7 +4,6 @@
 /gui/scripts/ne_10m_populated_places/
 /gui/scripts/ne_50m_admin_0_countries/
 /gui/scripts/ne_50m_admin_1_states_provinces_lines/
-/gui/scripts/ne_50m_populated_places/
 /gui/scripts/out/
 /build/
 /dist

diff --git a/gui/package.json b/gui/package.json
@@ -44,7 +44,6 @@
     "@types/d3-geo": "^1.11.0",
     "@types/enzyme": "^3.1.15",
     "@types/enzyme-adapter-react-16": "^1.0.3",
-    "@types/geojson": "^7946.0.7",
     "@types/mkdirp": "^0.5.2",
     "@types/mocha": "^5.2.6",
     "@types/node": "^10.12.3",

diff --git a/gui/scripts/README.md b/gui/scripts/README.md
@@ -28,7 +28,6 @@ or use cURL to download all ZIPs:
 curl -L -O https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/cultural/ne_50m_admin_0_countries.zip
 curl -L -O https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/cultural/ne_50m_admin_1_states_provinces_lines.zip
 curl -L -O https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_populated_places.zip
-curl -L -O https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/cultural/ne_50m_populated_places.zip
 ```
 
 Extract the downloaded ZIP files into `scripts` folder.
@@ -37,15 +36,13 @@ Make sure the following folders exist after extraction:
 - ne_50m_admin_0_countries
 - ne_50m_admin_1_states_provinces_lines
 - ne_10m_populated_places
-- ne_50m_populated_places
 
 or use the following script:
 
 ```
 unzip ne_50m_admin_0_countries.zip -d ne_50m_admin_0_countries/
 unzip ne_50m_admin_1_states_provinces_lines.zip -d ne_50m_admin_1_states_provinces_lines/
 unzip ne_10m_populated_places.zip -d ne_10m_populated_places/
-unzip ne_50m_populated_places.zip -d ne_50m_populated_places/
 ```
 
 ## Geo data extraction notes

diff --git a/gui/scripts/extract-geo-data.py b/gui/scripts/extract-geo-data.py
@@ -34,83 +34,13 @@
 # Relay locations gettext catalogue filename (.po)
 RELAY_LOCATIONS_PO_FILENAME = "relay-locations.po"
 
-# Countries gettext catalogue filename (.po)
-COUNTRIES_PO_FILENAME = "countries.po"
-
-# Cities gettext catalogue filename (.po)
-CITIES_PO_FILENAME = "cities.po"
-
-# The minimum population cap used to narrow down the cities dataset
-POPULATION_MAX_FILTER = 50000
-
 # Custom locale mapping between the identifiers in the app and Natural Earth datasets
 LOCALE_MAPPING = {
   # "zh" in Natural Earth Data referes to simplified chinese
   "zh-CN": "zh"
 }
 
 
-def extract_cities():
-  input_path = get_shape_path("ne_50m_populated_places")
-  output_path = path.join(OUT_DIR, "cities.json")
-
-  props_to_keep = frozenset(["scalerank", "name", "latitude", "longitude"])
-
-  features = []
-  with fiona.collection(input_path, "r") as source:
-    for feat in source:
-      props = lower_dict_keys(feat["properties"])
-
-      if props["pop_max"] >= POPULATION_MAX_FILTER:
-        for k in frozenset(props) - props_to_keep:
-          del props[k]
-
-        feat["properties"] = props
-        features.append(feat)
-
-  my_layer = {
-    "type": "FeatureCollection",
-    "features": features
-  }
-
-  with open(output_path, "w") as f:
-    f.write(json.dumps(my_layer))
-
-  print(c.green("Extracted data to {}".format(output_path)))
-
-
-def extract_countries():
-  input_path = get_shape_path("ne_50m_admin_0_countries")
-  output_path = path.join(OUT_DIR, "countries.json")
-
-  props_to_keep = frozenset(["name"])
-
-  features = []
-  with fiona.open(input_path) as source:
-    for feat in source:
-      geometry = feat["geometry"]
-
-      # convert country polygon to point
-      geometry.update(mapping(shape(geometry).representative_point()))
-
-      props = lower_dict_keys(feat["properties"])
-      for k in frozenset(props) - props_to_keep:
-        del props[k]
-
-      feat["properties"] = props
-      features.append(feat)
-
-  my_layer = {
-    "type": "FeatureCollection",
-    "features": features
-  }
-
-  with open(output_path, "w") as f:
-    f.write(json.dumps(my_layer))
-
-  print(c.green("Extracted data to {}".format(output_path)))
-
-
 def extract_geometry():
   input_path = get_shape_path("ne_50m_admin_0_countries")
   output_path = path.join(OUT_DIR, "geometry.json")
@@ -167,139 +97,6 @@ def extract_provinces_and_states_lines():
     print(c.red("geo2topo exited with {}. {}".format(p.returncode, errors.decode().strip())))
 
 
-def extract_countries_po():
-  input_path = get_shape_path("ne_50m_admin_0_countries")
-
-  for locale in os.listdir(LOCALE_DIR):
-    locale_dir = path.join(LOCALE_DIR, locale)
-    locale_out_dir = path.join(LOCALE_OUT_DIR, locale)
-
-    if os.path.isdir(locale_dir):
-      with fiona.open(input_path) as source:
-        po = POFile(encoding='utf-8', check_for_duplicates=True)
-        po.metadata = {"Content-Type": "text/plain; charset=utf-8"}
-        output_path = path.join(locale_out_dir, COUNTRIES_PO_FILENAME)
-
-        if not path.exists(locale_out_dir):
-          os.makedirs(locale_out_dir)
-
-        print("Generating {}".format(output_path))
-
-        for feat in source:
-          props = lower_dict_keys(feat["properties"])
-
-          name_key = "name_" + map_locale(locale)
-          name_fallback = "name"
-
-          country_name = props.get("name")
-          formal_country_name = props.get("formal_en", country_name)
-
-          if props.get(name_key) is not None:
-            translated_name = props.get(name_key)
-          elif props.get(name_fallback) is not None:
-            translated_name = props.get(name_fallback)
-            print(c.orange("Missing translation for {}".format(translated_name)))
-          else:
-            raise ValueError(
-              "Cannot find the translation for {}. Probe keys: {}"
-                .format(locale, (name_key, name_fallback))
-              )
-
-          entry = POEntry(
-            msgid=country_name,
-            msgstr=translated_name
-          )
-          po.append(entry)
-
-          # add additional record for the formal country name.
-          if country_name != formal_country_name and formal_country_name is not None:
-            entry = POEntry(
-              msgid=formal_country_name,
-              msgstr=translated_name
-            )
-            po.append(entry)
-
-          # exception for the US
-          if props.get("iso_a3") == "USA":
-            entry = POEntry(
-              msgid="USA",
-              msgstr=translated_name
-            )
-            po.append(entry)
-
-          # exception for the UK
-          if props.get("iso_a3") == "GBR":
-            entry = POEntry(
-              msgid="UK",
-              msgstr=translated_name
-            )
-            po.append(entry)
-
-        sort_pofile_entries(po)
-        po.save(output_path)
-        print(c.green("Extracted {} countries for {} to {}".format(len(po), locale, output_path)))
-
-
-def extract_cities_po():
-  input_path = get_shape_path("ne_50m_populated_places")
-  stats = []
-
-  for locale in os.listdir(LOCALE_DIR):
-    locale_dir = path.join(LOCALE_DIR, locale)
-    locale_out_dir = path.join(LOCALE_OUT_DIR, locale)
-
-    if os.path.isdir(locale_dir):
-      po = POFile(encoding='utf-8', check_for_duplicates=True)
-      po.metadata = {"Content-Type": "text/plain; charset=utf-8"}
-      output_path = path.join(locale_out_dir, CITIES_PO_FILENAME)
-      hits = 0
-      misses = 0
-
-      if not path.exists(locale_out_dir):
-        os.makedirs(locale_out_dir)
-
-      print("Generating {}".format(output_path))
-
-      with fiona.open(input_path) as source:
-        for feat in source:
-          props = lower_dict_keys(feat["properties"])
-
-          if props["pop_max"] >= POPULATION_MAX_FILTER:
-            name_key = "name_" + map_locale(locale)
-            name_fallback = "name"
-
-            if props.get(name_key) is not None:
-              translated_name = props.get(name_key)
-              hits += 1
-            elif props.get(name_fallback) is not None:
-              translated_name = props.get(name_fallback)
-              print(c.orange("Missing translation for {}".format(translated_name)))
-              misses += 1
-            else:
-              raise ValueError(
-                "Cannot find the translation for {}. Probe keys: {}"
-                  .format(locale, (name_key, name_fallback))
-                )
-
-            entry = POEntry(
-              msgid=props.get("name"),
-              msgstr=translated_name
-            )
-
-            try:
-              po.append(entry)
-            except ValueError as err:
-              print(c.orange("Cannot add an entry: {}".format(err)))
-
-      sort_pofile_entries(po)
-      po.save(output_path)
-      print(c.green("Extracted {} cities to {}".format(len(po), output_path)))
-
-      stats.append((locale, hits, misses))
-
-  print_stats_table("Cities translations", stats)
-
-
 def sort_pofile_entries(pofile):
   pofile.sort(key=lambda o: o.msgid_with_context)
 
@@ -678,14 +475,10 @@ def main():
     os.makedirs(LOCALE_OUT_DIR)
 
   # extract geo data
-  extract_cities()
-  extract_countries()
   extract_geometry()
   extract_provinces_and_states_lines()
 
   # extract translations
-  extract_countries_po()
-  extract_cities_po()
   extract_relay_translations()
 
 main()
diff --git a/gui/scripts/integrate-into-app.py b/gui/scripts/integrate-into-app.py
@@ -29,20 +29,12 @@
 
 # Geo assets for copying from generated content folder into the app folder
 GEO_ASSETS_TO_COPY = [
-  "cities.rbush.json",
-  "countries.rbush.json",
   "geometry.json",
   "geometry.rbush.json",
   "states-provinces-lines.json",
   "states-provinces-lines.rbush.json",
 ]
 
-# The filenames of gettext catalogues that should be copied as is
-TRANSLATIONS_TO_COPY = [
-  "cities.po",
-  "countries.po"
-]
-
 # The filenames of gettext catalogues that should be merged using msgcat
 TRANSLATIONS_TO_MERGE = [
   "relay-locations.po"
@@ -80,10 +72,7 @@ def merge_single_locale_folder(src, dst):
     src_po = path.join(src, f)
     dst_po = path.join(dst, f)
 
-    if f in TRANSLATIONS_TO_COPY:
-      print("Copying {} to {}".format(src_po, dst_po))
-      shutil.copyfile(src_po, dst_po)
-    elif f in TRANSLATIONS_TO_MERGE:
+    if f in TRANSLATIONS_TO_MERGE:
       # merge ../locales/*/file.po with ./out/locales/*/file.po
       # use existing translation to resolve conflicts
       merge_gettext_catalogues(dst_po, src_po)

diff --git a/gui/scripts/prepare-rtree.ts b/gui/scripts/prepare-rtree.ts
@@ -6,7 +6,6 @@
 import * as fs from 'fs';
 import * as path from 'path';
 import { Topology, GeometryCollection } from 'topojson-specification';
-import { GeoJSON } from 'geojson';
 import rbush from 'rbush';
 
 interface GeometryTopologyObjects {
@@ -16,7 +15,6 @@ interface GeometryTopologyObjects {
 
 function main() {
   const GEOMETRY_DATA_FILES = ['geometry', 'states-provinces-lines'];
-  const POINT_DATA_FILES = ['countries', 'cities'];
   const OUTPUT_DIR = path.join(__dirname, 'out');
 
   for (const name of GEOMETRY_DATA_FILES) {
@@ -29,17 +27,6 @@ function main() {
       console.error(`Failed to process ${name}: ${error.message}`);
     }
   }
-
-  for (const name of POINT_DATA_FILES) {
-    const source = path.join(OUTPUT_DIR, `${name}.json`);
-    const destination = path.join(OUTPUT_DIR, `${name}.rbush.json`);
-
-    try {
-      processPoints(source, destination);
-    } catch (error) {
-      console.error(`Failed to process ${name}: ${error.message}`);
-    }
-  }
 }
 
 function processGeometry(source: string, destination: string) {
@@ -70,35 +57,4 @@ function processGeometry(source: string, destination: string) {
   console.log(`Saved a rbush to ${destination}`);
 }
 
-function processPoints(source: string, destination: string) {
-  const collection = JSON.parse(fs.readFileSync(source, { encoding: 'utf8' })) as GeoJSON;
-
-  if (collection.type !== 'FeatureCollection') {
-    throw new Error(
-      `Invalid collection type ${collection.type} in ${source}. Expected FeatureCollection`,
-    );
-  }
-
-  const treeData = collection.features.map((feat) => {
-    if (feat.geometry.type !== 'Point') {
-      throw new Error(`Invalid geometry in ${source}. Expected "Point", got ${feat.geometry.type}`);
-    }
-
-    const { coordinates } = feat.geometry;
-    return {
-      ...feat,
-      minX: coordinates[0],
-      minY: coordinates[1],
-      maxX: coordinates[0],
-      maxY: coordinates[1],
-    };
-  });
-
-  const tree = rbush();
-  tree.load(treeData);
-  fs.writeFileSync(destination, JSON.stringify(tree.toJSON()));
-
-  console.log(`Saved a rbush to ${destination}`);
-}
-
 main();