Merge pull request crate#41 from crate/b/https

https support

CHANGES.txt

@@ -5,6 +5,8 @@ Changes for crate
 Unreleased
 ==========
 
+ - allow to specify https urls in client and crash cli
+
 2013/12/06 0.1.9
 ================
 

docs/client.txt

@@ -10,7 +10,7 @@ Before we can start we have to import the crate client::
     >>> from crate import client
 
 The client provides a ``connect()`` function which is used to establish a
-connection, the first argument is the server to connect to::
+connection, the first argument is the url of the server to connect to::
 
     >>> connection = client.connect(crate_host)
 
@@ -19,14 +19,14 @@ In order for clients to make use of this property it is recommended to specify
 all hosts of the cluster. This way if a server does not respond, the request is
 automatically routed to the next server::
 
-    >>> invalid_host = 'not_responding_host:9200'
+    >>> invalid_host = 'http://not_responding_host:9200'
     >>> connection = client.connect([invalid_host, crate_host])
 
-If no ``servers`` are given, the default one ``127.0.0.1:9200`` is used::
+If no ``servers`` are given, the default one ``http://127.0.0.1:9200`` is used::
 
     >>> connection = client.connect()
     >>> connection.client._active_servers
-    ['127.0.0.1:9200']
+    ['http://127.0.0.1:9200']
 
 It's possible to define a default timeout value in seconds for all servers
 using the optional parameter ``timeout``::

setup.py

@@ -9,6 +9,7 @@
     'requests>=2.0.1,<3.0',
     'PrettyTable>=0.7,<0.8',
     'appdirs>=1.2,<2.0',
+    'six'
 ]
 
 if (2, 6) == sys.version_info[:2]:

src/crate/client/connection.py

@@ -60,6 +60,6 @@ def connect(servers=None, timeout=None, client=None):
         client used to communicate with crate.
 
     >>> connect(['host1:9200', 'host2:9200'])
-    <Connection <Client ['host1:9200', 'host2:9200']>>
+    <Connection <Client ['http://host1:9200', 'http://host2:9200']>>
     """
     return Connection(servers=servers, timeout=timeout, client=client)

src/crate/client/crash.py

@@ -37,8 +37,6 @@ class CrateCmd(Cmd):
 
     def __init__(self, stdin=None, stdout=None):
         Cmd.__init__(self, "tab", stdin, stdout)
-        self.conn = client.connect()
-        self.cursor = self.conn.cursor()
         self.partial_lines = []
 
     def do_connect(self, server):
@@ -57,7 +55,7 @@ def do_connect(self, server):
                 results.append(
                     server_infos + (True, "OK", )
                 )
-        self.pprint(results, ["host", "node_name", "connected", "message"])
+        self.pprint(results, ["server_url", "node_name", "connected", "message"])
         if failed == len(results):
             self.print_error("connect")
         else:
@@ -328,8 +326,7 @@ def main():
     atexit.register(readline.write_history_file, history_file_path)
 
     cmd = CrateCmd()
-    if args.hosts:
-        cmd.do_connect(args.hosts)
+    cmd.do_connect(args.hosts)
 
     # select.select on sys.stdin doesn't work on windows
     # so currently there is no pipe support

src/crate/client/crash.txt

@@ -15,40 +15,41 @@ When you connect to a server that is not reachable or whose hostname cannot be r
 you will get an error::
 
     cr> connect 127.0.0.1:65535
-    +-----------------+-----------+-----------+--------------------+
-    | host            | node_name | connected | message            |
-    +-----------------+-----------+-----------+--------------------+
-    | 127.0.0.1:65535 | NULL      | FALSE     | Connection refused |
-    +-----------------+-----------+-----------+--------------------+
+    +------------------------+-----------+-----------+--------------------+
+    | server_url             | node_name | connected | message            |
+    +------------------------+-----------+-----------+--------------------+
+    | http://127.0.0.1:65535 | NULL      | FALSE     | Connection refused |
+    +------------------------+-----------+-----------+--------------------+
     CONNECT ERROR
 
-    cr> connect non-existent.lol:9200
-    +-----------------------+-----------+-----------+--------------------------------+
-    | host                  | node_name | connected | message                        |
-    +-----------------------+-----------+-----------+--------------------------------+
-    | non-existent.lol:9200 | NULL      | FALSE     | Hostname could no be resolved. |
-    +-----------------------+-----------+-----------+--------------------------------+
+    cr> connect null.lol:9200
+    +----------------------+-----------+-----------+--------------------------------+
+    | server_url           | node_name | connected | message                        |
+    +----------------------+-----------+-----------+--------------------------------+
+    | http://null.lol:9200 | NULL      | FALSE     | Hostname could no be resolved. |
+    +----------------------+-----------+-----------+--------------------------------+
     CONNECT ERROR
 
+
 Successful connects will give you some information about the servers you connect to::
 
     cr> connect 127.0.0.1:9295
-    +----------------+-----------+-----------+---------+
-    | host           | node_name | connected | message |
-    +----------------+-----------+-----------+---------+
-    | 127.0.0.1:9295 | crate     | TRUE      | OK      |
-    +----------------+-----------+-----------+---------+
+    +-----------------------+-----------+-----------+---------+
+    | server_url            | node_name | connected | message |
+    +-----------------------+-----------+-----------+---------+
+    | http://127.0.0.1:9295 | crate     | TRUE      | OK      |
+    +-----------------------+-----------+-----------+---------+
     CONNECT OK
 
 If you connect to more than one server, the command will succeed
 if at least one server is reachable::
 
-    cr> CONNECT 127.0.0.1:9295 non-existent.lol:9295
+    cr> CONNECT 127.0.0.1:9295 null.lol:9295
     +-----------------------+-----------+-----------+--------------------------------+
-    | host                  | node_name | connected | message                        |
+    | server_url            | node_name | connected | message                        |
     +-----------------------+-----------+-----------+--------------------------------+
-    | 127.0.0.1:9295        | crate     | TRUE      | OK                             |
-    | non-existent.lol:9295 | NULL      | FALSE     | Hostname could no be resolved. |
+    | http://127.0.0.1:9295 | crate     | TRUE      | OK                             |
+    | http://null.lol:9295  | NULL      | FALSE     | Hostname could no be resolved. |
     +-----------------------+-----------+-----------+--------------------------------+
     CONNECT OK
 

src/crate/client/http.py

@@ -5,9 +5,8 @@
 import sys
 from time import time
 import threading
-
+from six.moves.urllib.parse import urlparse
 import requests
-
 from crate.client.exceptions import (
     ConnectionError, DigestNotFoundException, ProgrammingError, BlobsDisabledException)
 
@@ -29,21 +28,46 @@ class Client(object):
     retry_interval = 30
     """Retry interval for failed servers in seconds."""
 
-    default_server = "127.0.0.1:9200"
+    default_server = "http://127.0.0.1:9200"
     """Default server to use if no servers are given on instantiation."""
 
     def __init__(self, servers=None, timeout=None):
         if not servers:
-            servers = self.default_server
-        if isinstance(servers, basestring):
-            servers = servers.split()
-        self._active_servers = list(servers)
+            servers = [self.default_server]
+        else:
+            if isinstance(servers, basestring):
+                servers = servers.split()
+            servers = [self._server_url(s) for s in servers]
+        self._active_servers = servers
         self._http_timeout = timeout
         self._inactive_servers = []
         self._lock = threading.RLock()
         self._local = threading.local()
         self._session = requests.session()
 
+    @staticmethod
+    def _server_url(server):
+        """
+        Normalizes a given server string to an url
+
+        >>> print(Client._server_url('a'))
+        http://a
+        >>> print(Client._server_url('a:9345'))
+        http://a:9345
+        >>> print(Client._server_url('https://a:9345'))
+        https://a:9345
+        >>> print(Client._server_url('https://a'))
+        https://a
+        >>> print(Client._server_url('demo.crate.io'))
+        http://demo.crate.io
+        """
+        parsed = urlparse(server)
+        if parsed.path and not parsed.netloc and not parsed.scheme:
+            return 'http://%s' % parsed.path
+        url = '%s://%s' % (parsed.scheme, parsed.netloc)
+        return url
+
+
     def sql(self, stmt, parameters=None):
         """
         Execute SQL stmt against the crate server.
@@ -158,9 +182,9 @@ def _request(self, method, path, **kwargs):
                     raise ProgrammingError(e.response.content)
                 raise ProgrammingError()
 
-    def _do_request(self, server, method, path, **kwargs):
+    def _do_request(self, url, method, path, **kwargs):
         """do the actual request to a chosen server"""
-        uri = "http://{server}/{path}".format(server=server, path=path)
+        uri = "{url}/{path}".format(url=url, path=path)
         return self._session.request(method, uri, timeout=self._http_timeout, **kwargs)
 
     def _raise_for_status(self, response):

src/crate/client/http.txt

@@ -18,28 +18,28 @@ is used::
 
     >>> http_client = HttpClient()
     >>> http_client._active_servers
-    ['127.0.0.1:9200']
+    ['http://127.0.0.1:9200']
 
 When using a list of servers, the servers are selected by round-robin::
 
     >>> invalid_host = "invalid_host:9999"
     >>> even_more_invalid_host = "even_more_invalid_host:9999"
     >>> http_client = HttpClient([crate_host, invalid_host, even_more_invalid_host])
     >>> http_client._get_server()
-    '127.0.0.1:9295'
+    'http://127.0.0.1:9295'
 
     >>> http_client._get_server()
-    'invalid_host:9999'
+    'http://invalid_host:9999'
 
     >>> http_client._get_server()
-    'even_more_invalid_host:9999'
+    'http://even_more_invalid_host:9999'
 
 Servers with connection errors will be removed from the active server list::
 
     >>> http_client = HttpClient([invalid_host, even_more_invalid_host, crate_host])
     >>> result = http_client.sql('select name from locations')
     >>> http_client._active_servers
-    ['127.0.0.1:9295']
+    ['http://127.0.0.1:9295']
 
 Inactive servers will be re-added after a given time interval.
 To validate this, set the interval very short and sleep for that interval::
@@ -48,7 +48,9 @@ To validate this, set the interval very short and sleep for that interval::
     >>> import time; time.sleep(1)
     >>> result = http_client.sql('select name from locations')
     >>> http_client._active_servers
-    ['invalid_host:9999', 'even_more_invalid_host:9999', '127.0.0.1:9295']
+    ['http://invalid_host:9999',
+     'http://even_more_invalid_host:9999',
+     'http://127.0.0.1:9295']
 
 If no active servers are available and the retry interval is not reached, just use the oldest
 inactive one::
@@ -57,7 +59,7 @@ inactive one::
     >>> result = http_client.sql('select name from locations')
     >>> http_client._active_servers = []
     >>> http_client._get_server()
-    'invalid_host:9999'
+    'http://invalid_host:9999'
 
 
 SQL Statements

src/crate/client/sqlalchemy/test.py

@@ -29,14 +29,14 @@ def setUp(self):
     def test_default_connection(self):
         engine = sa.create_engine('crate://')
         conn = engine.raw_connection()
-        self.assertEquals("<Connection <Client ['127.0.0.1:9200']>>",
+        self.assertEquals("<Connection <Client ['http://127.0.0.1:9200']>>",
                           repr(conn.connection))
 
     def test_connection_server(self):
         engine = sa.create_engine(
             "crate://otherhost:19201")
         conn = engine.raw_connection()
-        self.assertEquals("<Connection <Client ['otherhost:19201']>>",
+        self.assertEquals("<Connection <Client ['http://otherhost:19201']>>",
                           repr(conn.connection))
 
     def test_connection_multiple_server(self):
@@ -47,7 +47,7 @@ def test_connection_multiple_server(self):
         )
         conn = engine.raw_connection()
         self.assertEquals(
-            "<Connection <Client ['localhost:9201', 'localhost:9202']>>",
+            "<Connection <Client ['http://localhost:9201', 'http://localhost:9202']>>",
             repr(conn.connection))
 
 

src/crate/client/test_http.py

@@ -74,16 +74,18 @@ def test_http_error_is_re_raised_in_raise_for_status(self):
 
     def test_connect(self):
         client = Client(servers="localhost:9200 localhost:9201")
-        self.assertEqual(client._active_servers, ["localhost:9200", "localhost:9201"])
+        self.assertEqual(client._active_servers,
+                         ["http://localhost:9200", "http://localhost:9201"])
 
         client = Client(servers="localhost:9200")
-        self.assertEqual(client._active_servers, ["localhost:9200"])
+        self.assertEqual(client._active_servers, ["http://localhost:9200"])
 
         client = Client(servers=["localhost:9200"])
-        self.assertEqual(client._active_servers, ["localhost:9200"])
+        self.assertEqual(client._active_servers, ["http://localhost:9200"])
 
         client = Client(servers=["localhost:9200", "127.0.0.1:9201"])
-        self.assertEqual(client._active_servers, ["localhost:9200", "127.0.0.1:9201"])
+        self.assertEqual(client._active_servers,
+                         ["http://localhost:9200", "http://127.0.0.1:9201"])
 
 
 class ThreadSafeHttpClientTest(TestCase):
@@ -103,13 +105,16 @@ class ThreadSafeHttpClientTest(TestCase):
     thread_timeout = 5.0  # seconds
 
     def __init__(self, *args, **kwargs):
-        self.client = Client(self.servers)
-        self.client.retry_interval = 0.0001  # faster retry
-        self.client._session = FakeSessionFailSometimes()  # patch the clients session
         self.event = Event()
         self.err_queue = queue.Queue()
         super(ThreadSafeHttpClientTest, self).__init__(*args, **kwargs)
 
+    def setUp(self):
+        super(ThreadSafeHttpClientTest, self).setUp()
+        self.client = Client(self.servers)
+        self.client.retry_interval = 0.0001  # faster retry
+        self.client._session = FakeSessionFailSometimes()  # patch the clients session
+
     def _run(self):
         self.event.wait()  # wait for the others
         expected_num_servers = len(self.servers)
@@ -199,10 +204,11 @@ class KeepAliveClientTest(TestCase):
     def __init__(self, *args, **kwargs):
         super(KeepAliveClientTest, self).__init__(*args, **kwargs)
         self.server_process = Process(target=self._run_server)
-        self.client = Client(["%s:%d" % self.server_address])
+
 
     def setUp(self):
         super(KeepAliveClientTest, self).setUp()
+        self.client = Client(["%s:%d" % self.server_address])
         self.server_process.start()
         time.sleep(.10)
 

src/crate/client/tests.py

@@ -178,6 +178,7 @@ def test_suite():
     suite.addTest(unittest.makeSuite(ThreadSafeHttpClientTest))
     suite.addTest(sqlalchemy_tests)
     suite.addTest(doctest.DocTestSuite('crate.client.connection'))
+    suite.addTest(doctest.DocTestSuite('crate.client.http'))
 
     s = doctest.DocFileSuite(
         'sqlalchemy/itests.txt',

versions.cfg

@@ -12,7 +12,7 @@ lovely.buildouthttp = 0.5.0
 
 # Required by:
 # zope.testrunner==4.4.1
-six = 1.3.0
+six = 1.4.1
 
 # Required by:
 # zope.testrunner==4.4.1
@@ -35,3 +35,7 @@ prettytable = 0.7.2
 mock = 1.0.1
 argparse = 1.2.1
 SQLAlchemy = 0.8.2
+
+appdirs = 1.2.0
+readline = 6.2.4.1
+zc.customdoctests = 1.0.1