Merge branch 'py3' into 'master'

Initial Python3 support

Closes #1

See merge request tpo/network-health/exitmap!1

bin/exitmap

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # Copyright 2013, 2014, 2016 Philipp Winter <[email protected]>
 #

requirements.txt

@@ -1,2 +1,3 @@
 dnspython
 stem
+pysocks

src/exitmap.py

@@ -28,7 +28,7 @@
 import datetime
 import random
 import logging
-import ConfigParser
+from configparser import ConfigParser
 import functools
 import pwd
 
@@ -114,7 +114,7 @@ def parse_cmd_args():
         home_dir = os.path.expanduser("~")
         config_file = os.path.join(home_dir, ".exitmaprc")
 
-    config_parser = ConfigParser.SafeConfigParser()
+    config_parser = ConfigParser()
     file_parsed = config_parser.read([config_file])
     if file_parsed:
         try:

src/modules/checktest.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # Copyright 2013-2017 Philipp Winter <[email protected]>
 #
@@ -24,10 +24,9 @@
 import sys
 import json
 import logging
-try:
-    import urllib2
-except ImportError:
-    import urllib.request as urllib2
+import urllib.request
+import socks
+import socket
 
 from util import exiturl
 
@@ -50,10 +49,10 @@ def fetch_page(exit_desc):
     url = exiturl(exit_desc.fingerprint)
 
     try:
-        data = urllib2.urlopen("https://check.torproject.org/api/ip",
+        data = urllib.request.urlopen("https://check.torproject.org/api/ip",
                                timeout=10).read()
     except Exception as err:
-        log.debug("urllib2.urlopen says: %s" % err)
+        log.debug("urllib.request.urlopen says: %s" % err)
         return
 
     if not data:

src/modules/cloudflared.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # Copyright 2016 Philipp Winter <[email protected]>
 #
@@ -22,9 +22,9 @@
 """
 
 import sys
-import StringIO
+import io
 import gzip
-import httplib
+import http.client
 import collections
 import logging
 
@@ -35,7 +35,7 @@
 destinations = [("www.cloudflare.com", 443)]
 DOMAIN, PORT = destinations[0]
 
-CAPTCHA_SIGN = "Attention Required! | Cloudflare"
+CAPTCHA_SIGN = b"Attention Required! | Cloudflare"
 
 # Mimic Tor Browser's request headers, so CloudFlare won't return a 403 because
 # it thinks we are a bot.
@@ -57,7 +57,7 @@ def decompress(data):
     """
 
     try:
-        buf = StringIO.StringIO(data)
+        buf = io.StringIO(data)
         fileobj = gzip.GzipFile(fileobj=buf)
         data = fileobj.read()
     except Exception:
@@ -74,7 +74,7 @@ def is_cloudflared(exit_fpr):
     exit_url = util.exiturl(exit_fpr)
     log.debug("Probing exit relay \"%s\"." % exit_url)
 
-    conn = httplib.HTTPSConnection(DOMAIN, PORT, strict=False)
+    conn = http.client.HTTPSConnection(DOMAIN, PORT)
     conn.request("GET", "/", headers=collections.OrderedDict(HTTP_HEADERS))
     try:
         response = conn.getresponse()

src/modules/dnspoison.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # Copyright 2013-2017 Philipp Winter <[email protected]>
 #
@@ -50,7 +50,7 @@ def setup():
 
     log.debug("Populating domain dictionary.")
 
-    for domain in domains.iterkeys():
+    for domain in list(domains.keys()):
         response = dns.resolver.query(domain)
         for record in response:
             log.debug("Domain %s maps to %s." % (domain, record.address))
@@ -98,7 +98,7 @@ def probe(exit_desc, run_python_over_tor, run_cmd_over_tor, **kwargs):
     Probe the given exit relay and check if all domains resolve as expected.
     """
 
-    for domain in domains.iterkeys():
+    for domain in list(domains.keys()):
         run_python_over_tor(resolve, exit_desc, domain, domains[domain])
 
 

src/modules/dnssec.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # Copyright 2016 Philipp Winter <[email protected]>
 #

src/modules/patchingCheck.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # Copyright 2014-2016 Philipp Winter <[email protected]>
 # Copyright 2014 Josh Pitts <[email protected]>
@@ -36,7 +36,7 @@
 import sys
 import os
 try:
-    import urllib2
+    import urllib.request, urllib.error, urllib.parse
 except ImportError:
     import urllib.request as urllib2
 import tempfile
@@ -81,15 +81,15 @@ def setup():
 
     log.info("Creating temporary reference files.")
 
-    for url, _ in check_files.iteritems():
+    for url, _ in check_files.items():
 
         log.debug("Attempting to download <%s>." % url)
 
-        request = urllib2.Request(url)
+        request = urllib.request.Request(url)
         request.add_header('User-Agent', test_agent)
 
         try:
-            data = urllib2.urlopen(request).read()
+            data = urllib.request.urlopen(request).read()
         except Exception as err:
             log.warning("urlopen() failed: %s" % err)
 
@@ -111,7 +111,7 @@ def teardown():
 
     log.info("Removing reference files.")
 
-    for _, file_info in check_files.iteritems():
+    for _, file_info in check_files.items():
 
         orig_file, _ = file_info
         log.info("Removing file \"%s\"." % orig_file)
@@ -161,19 +161,19 @@ def run_check(exit_desc):
 
     exiturl = util.exiturl(exit_desc.fingerprint)
 
-    for url, file_info in check_files.iteritems():
+    for url, file_info in check_files.items():
 
         orig_file, orig_digest = file_info
 
         log.debug("Attempting to download <%s> over %s." % (url, exiturl))
 
         data = None
 
-        request = urllib2.Request(url)
+        request = urllib.request.Request(url)
         request.add_header('User-Agent', test_agent)
 
         try:
-            data = urllib2.urlopen(request, timeout=20).read()
+            data = urllib.request.urlopen(request, timeout=20).read()
         except Exception as err:
             log.warning("urlopen() failed for %s: %s" % (exiturl, err))
             continue

src/modules/rtt.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # Copyright 2013-2016 Philipp Winter <[email protected]>
 # Copyright 2016 Zack Weinberg <[email protected]>
@@ -231,7 +231,7 @@ def choose_probe_order(dests):
     remaining = {}
     last_appearance = {}
     full_address = {}
-    for host, usable_ports in hosts.iteritems():
+    for host, usable_ports in hosts.items():
         for p in PREFERRED_PORT_ORDER:
             if p in usable_ports:
                 full_address[host] = (host, p)
@@ -241,7 +241,7 @@ def choose_probe_order(dests):
     rv = []
     deadcycles = 0
     while remaining:
-        ks = remaining.keys()
+        ks = list(remaining.keys())
         x = random.choice(ks)
         last = last_appearance[x]
         if last == -1 or (len(rv) - last) >= (len(ks) // 4):

src/modules/testfds.py

@@ -25,14 +25,12 @@
 import sys
 import re
 import logging
-try:
-    import urllib2
-except ImportError:
-    import urllib.request as urllib2
+import urllib.request, urllib.error, urllib.parse
 
 from util import exiturl
 
 import stem.descriptor.server_descriptor as descriptor
+import socks
 
 log = logging.getLogger(__name__)
 
@@ -50,10 +48,10 @@ def fetch_page(exit_desc):
 
     data = None
     try:
-        data = urllib2.urlopen("https://people.torproject.org/~phw/check_file",
-                               timeout=10).read()
+        data = urllib.request.urlopen("https://people.torproject.org/~phw/check_file",
+                               timeout=10).read().decode("utf-8")
     except Exception as err:
-        log.warning("urllib2.urlopen for %s says: %s." %
+        log.warning("urllib.request.urlopen for %s says: %s." %
                     (exit_desc.fingerprint, err))
         return
 

src/relayselector.py

@@ -178,7 +178,7 @@ def __init__(self):
 
     exit_candidates = [
         desc
-        for fpr, desc in have_exit_policy.iteritems()
+        for fpr, desc in have_exit_policy.items()
         if stem.Flag.EXIT in cached_consensus.get(fpr, stub_desc).flags
     ]