Generate tsv instead of txt via discussion in gitter

weiki516 · Dec 17, 2015 · 4f4c8eb · 4f4c8eb
1 parent f267fc9
commit 4f4c8eb
Show file tree

Hide file tree

Showing 36 changed files with 59,879 additions and 59,855 deletions.
diff --git a/2002.txt b/2002.txt
diff --git a/200212.tsv b/200212.tsv
diff --git a/2003.txt b/2003.txt
diff --git a/200306.tsv b/200306.tsv
diff --git a/200306.txt b/200306.txt
diff --git a/200312.tsv b/200312.tsv
diff --git a/2004.txt b/2004.txt
diff --git a/200403.tsv b/200403.tsv
diff --git a/200403.txt b/200403.txt
diff --git a/200409.tsv b/200409.tsv
diff --git a/200409.txt b/200409.txt
diff --git a/200412.tsv b/200412.tsv
diff --git a/2005.txt b/2005.txt
diff --git a/200506.tsv b/200506.tsv
diff --git a/200506.txt b/200506.txt
diff --git a/200512.tsv b/200512.tsv
diff --git a/2006.txt b/2006.txt
diff --git a/200612.tsv b/200612.tsv
diff --git a/2007.txt b/2007.txt
diff --git a/200712.tsv b/200712.tsv
diff --git a/2008.txt b/2008.txt
diff --git a/200812.tsv b/200812.tsv
diff --git a/2009.txt b/2009.txt
diff --git a/200912.tsv b/200912.tsv
diff --git a/2010.txt b/2010.txt
diff --git a/201010.tsv b/201010.tsv
diff --git a/2011.txt b/2011.txt
diff --git a/201110.tsv b/201110.tsv
diff --git a/2012.txt b/2012.txt
diff --git a/201210.tsv b/201210.tsv
diff --git a/2013.txt b/2013.txt
diff --git a/201308.tsv b/201308.tsv
diff --git a/2014.txt b/2014.txt
diff --git a/201410.tsv b/201410.tsv
diff --git a/revisions.json b/revisions.json
@@ -1,25 +1,25 @@
 {
   "GB2260": [
-    "2007",
-    "2002"
+    "200712",
+    "200212"
   ],
   "revisions": [
-    "2014",
-    "2013",
-    "2012",
-    "2011",
-    "2010",
-    "2009",
-    "2008",
-    "2007",
-    "2006",
-    "2005",
+    "201410",
+    "201308",
+    "201210",
+    "201110",
+    "201010",
+    "200912",
+    "200812",
+    "200712",
+    "200612",
+    "200512",
     "200506",
-    "2004",
+    "200412",
     "200409",
     "200403",
-    "2003",
+    "200312",
     "200306",
-    "2002"
+    "200212"
   ]
 }
diff --git a/scripts/parse.py b/scripts/parse.py
@@ -14,24 +14,24 @@
 
 URL_BASE = 'http://www.stats.gov.cn/tjsj/tjbz/xzqhdm/'
 URL_LIST = [
-    # (year, url, is_mass)
-    (2014, '201504/t20150415_712722.html', False),
-    (2013, '201401/t20140116_501070.html', False),
-    (2012, '201301/t20130118_38316.html', False),
-    (2011, '201201/t20120105_38315.html', False),
-    (2010, '201107/t20110726_38314.html', False),
-    (2009, '201006/t20100623_38313.html', False),
-    (2008, '200906/t20090626_38312.html', False),
-    (2007, '200802/t20080215_38311.html', False),
-    (2006, '200704/t20070411_38310.html', False),
-    (2005, '200410/t20041022_38307.html', True),
+    # (revision, url, is_mass)
+    (201410, '201504/t20150415_712722.html', False),
+    (201308, '201401/t20140116_501070.html', False),
+    (201210, '201301/t20130118_38316.html', False),
+    (201110, '201201/t20120105_38315.html', False),
+    (201010, '201107/t20110726_38314.html', False),
+    (200912, '201006/t20100623_38313.html', False),
+    (200812, '200906/t20090626_38312.html', False),
+    (200712, '200802/t20080215_38311.html', False),
+    (200612, '200704/t20070411_38310.html', False),
+    (200512, '200410/t20041022_38307.html', True),
     (200506, '200410/t20041022_38306.html', True),
-    (2004, '200410/t20041022_38305.html', True),
+    (200412, '200410/t20041022_38305.html', True),
     (200409, '200410/t20041022_38304.html', True),
     (200403, '200406/t20040607_38302.html', True),
-    (2003, '200402/t20040211_38301.html', True),
+    (200312, '200402/t20040211_38301.html', True),
     (200306, '200307/t20030722_38300.html', True),
-    (2002, '200302/t20030219_38299.html', True),
+    (200212, '200302/t20030219_38299.html', True),
 ]
 
 XPATH_EXPRS = [
@@ -42,6 +42,8 @@
     './/p[@class="MsoNormal"]//span//text()',
 ]
 
+SAC = [200212, 200712]
+
 
 def strip_spaces_in_chinese_words(line):
     cjk_chars = u'\u3007\u4e00-\u9fff\u3400-\u4dbf\uf900-\ufaff'
@@ -93,15 +95,20 @@ def main():
         print('Usage: %s [dir]' % sys.argv[0], file=sys.stderr)
         sys.exit(0)
 
-    for suffix, url, is_mass in URL_LIST:
+    for revision, url, is_mass in URL_LIST:
         req = requests.get(URL_BASE + url)
         req.encoding = 'utf-8'
         el = fromstring(req.text)
 
-        dirname = os.path.join(sys.argv[1], '%s.txt' % suffix)
+        dirname = os.path.join(sys.argv[1], '%s.tsv' % revision)
         print('--> %s' % dirname, file=sys.stderr)
 
+        source = 'stats'
+        if revision in SAC:
+            source = 'sac'
+
         with open(dirname, 'w') as dest_file:
+            print(b'Source\tRevision\tCode\tName', file=dest_file)
             for line in iter_lines(el, is_mass):
                 text = strip_spaces_in_chinese_words(strip_comments(line))
                 if not text:
@@ -113,7 +120,7 @@ def main():
                 except ValueError:
                     print('ignored: %s' % text, file=sys.stderr)
                 else:
-                    out = '%s\t%s' % (code, name)
+                    out = '%s\t%s\t%s\t%s' % (source, revision, code, name)
                     print(out.encode('utf-8'), file=dest_file)