Some fixes for whois loader

darcs-hash:20090515084018-f1522-f76fc53b8236dc6a07b589802178de35465b1a0c.gz

utilities/whois.py

@@ -41,19 +41,23 @@
 
 config.parse_options()
 
-pyflaglog.start_log_thread()
+#pyflaglog.start_log_thread()
 
 dbh = DB.DBO()
 def print_address(address):
-    whois_id = Whois.lookup_whois(address)
+    whois_id = Whois.lookup_whois_id(dbh, address)
+    if not whois_id:
+        print "IP Address %s not found (%s)" % (address, whois_id)
+        return
+
     dbh.execute("SELECT INET_NTOA(start_ip) as start_ip, netname, numhosts, country, descr, remarks, adminc, techc, status from whois where id=%s limit 1",whois_id)
     row = dbh.fetch()
     row['ip'] = address
     row['descr'] = '\ndescr:          '.join([ x for x in row['descr'].splitlines()])
     print "------ %(ip)s ------\nnetname:        %(netname)s\nCountry:        %(country)s\ninetnum:        %(start_ip)s\nhosts:          %(numhosts)s\ndescr:          %(descr)s\n" % row
 
     try:
-        if config.geoip_display:
+        if 0 and config.geoip_display:
             print Whois.geoip_cached_record(address)
             print Whois.get_all_geoip_data(address)
     except: pass

utilities/whois_load.py

@@ -32,7 +32,7 @@
 # Only have 'RIR' stats for lacnic and arin
 # ...though you may be able to request full databases from them
 
-urls = {'apnic':'ftp://ftp.apnic.net/apnic/whois-data/APNIC/split/apnic.db.inetnum.gz',
+urls = {#'apnic':'ftp://ftp.apnic.net/apnic/whois-data/APNIC/split/apnic.db.inetnum.gz',
         'ripe':'ftp://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz',
         'arin':'ftp://ftp.arin.net/pub/stats/arin/delegated-arin-latest',
         'lacnic':'ftp://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-latest'}
@@ -139,9 +139,9 @@ def parse_whois(self, string):
       self.netname = self._getsingle('netname', string)
       self.country = self._getsingle('country', string)
       self.adminc = self._getsingle('adminc', string)
-      self.techc = self._getsingle('techc', string)
-      self.descr = self._getmulti('descr', string)
-      self.remarks = self._getmulti('remarks', string)
+      self.techc = self._getsingle('techc', string).decode("UTF8","ignore")
+      self.descr = self._getmulti('descr', string).decode("UTF8","ignore")
+      self.remarks = self._getmulti('remarks', string).decode("UTF8","ignore")
 
       # get status
       status_str = self._getsingle('status', string).lower()
@@ -349,11 +349,13 @@ def __iter__(self):
 dbh.mass_insert_start('whois')
 
 for k,url in urls.items():
-  if not getattr(config, k): continue
-
-  db = Whois(url)
-  if not db:
-    print "Invalid url: %s" % url
+#  if not getattr(config, k): continue
+
+  print "Processing %s" % url
+  try:
+    db = Whois(url)
+  except Exception, e:
+    print "Error %s" % e
     continue
 
   # add this source to db
@@ -378,7 +380,7 @@ def __iter__(self):
       adminc = rec.adminc[:250],
       techc = rec.techc[:250],
       id = whois_id,
-      descr = rec.descr,
+      __descr = rec.descr,
       remarks = rec.remarks,
       status = rec.status)