Revamp the library for 2.0.0

.gitignore

@@ -8,7 +8,6 @@ __pycache__/
 
 # Distribution / packaging
 .Python
-env/
 build/
 develop-eggs/
 dist/
@@ -20,9 +19,11 @@ lib64/
 parts/
 sdist/
 var/
+wheels/
 *.egg-info/
 .installed.cfg
 *.egg
+MANIFEST
 
 # PyInstaller
 #  Usually these files are written by a python script from a template
@@ -42,8 +43,9 @@ htmlcov/
 .cache
 nosetests.xml
 coverage.xml
-*,cover
+*.cover
 .hypothesis/
+.pytest_cache/
 
 # Translations
 *.mo
@@ -52,6 +54,7 @@ coverage.xml
 # Django stuff:
 *.log
 local_settings.py
+db.sqlite3
 
 # Flask stuff:
 instance/
@@ -66,7 +69,7 @@ docs/_build/
 # PyBuilder
 target/
 
-# IPython Notebook
+# Jupyter Notebook
 .ipynb_checkpoints
 
 # pyenv
@@ -75,17 +78,36 @@ target/
 # celery beat schedule file
 celerybeat-schedule
 
-# dotenv
-.env
+# SageMath parsed files
+*.sage.py
 
-# virtualenv
+# Environments
+.env
+.venv
+env/
 venv/
 ENV/
+env.bak/
+venv.bak/
 
 # Spyder project settings
 .spyderproject
+.spyproject
 
 # Rope project settings
 .ropeproject
 
-.idea*
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+# MacOS
+.DS_Store
+
+# PyCharm
+.idea/
+
+# KQ specific
+output.log

.travis.yml

@@ -1,17 +1,19 @@
-language: python
 sudo: false
+language: python
 python:
-    - 2.7
-    - 3.4
     - 3.5
     - 3.6
+services:
+    - docker
+before_install:
+    - docker run --name kafka -d -p 2181:2181 -p 9092:9092 --env ADVERTISED_HOST=127.0.0.1 --env ADVERTISED_PORT=9092 spotify/kafka
 install:
-    - pip install coverage
-    - pip install pytest
-    - pip install pytest-cov
-    - pip install python-coveralls
-    - python setup.py install
+    - pip install flake8 mock pytest pytest-cov python-coveralls sphinx sphinx_rtd_theme
+    - pip install .
 script:
-    - py.test --cov=kq
+    - python -m flake8
+    - python -m sphinx -b doctest docs docs/_build
+    - python -m sphinx -b html -W docs docs/_build
+    - py.test -s -v --cov=kq
 after_success:
-    - coveralls
+    - coveralls

MANIFEST.in

@@ -0,0 +1,2 @@
+include README.rst LICENSE
+prune tests

README.rst

@@ -13,7 +13,7 @@ KQ: Kafka-based Job Queue for Python
     :target: https://badge.fury.io/py/kq
     :alt: Package Version
 
-.. image:: https://img.shields.io/badge/python-2.7%2C%203.4%2C%203.5%2C%203.6-blue.svg
+.. image:: https://img.shields.io/badge/python-3.5%2C%203.6-blue.svg
     :target: https://github.com/joowani/kq
     :alt: Python Versions
 
@@ -31,117 +31,138 @@ KQ: Kafka-based Job Queue for Python
 
 |
 
-KQ (Kafka Queue) is a lightweight Python library which provides a simple API
-to process jobs asynchronously in the background. It uses `Apache Kafka`_ and
-is designed primarily for ease of use.
+**KQ (Kafka Queue)** is a lightweight Python library which lets you queue and
+execute jobs asynchronously using `Apache Kafka`_. It uses kafka-python_ under
+the hood.
 
-.. _Apache Kafka: https://kafka.apache.org
+Announcements
+=============
 
+* KQ version `2.0.0`_ is now out!
+* Please see the releases_ page for latest updates.
 
 Requirements
 ============
 
-- Apache Kafka 0.9+
-- Python 2.7, 3.4, 3.5 or 3.6
-
+* `Apache Kafka`_ 0.9+
+* Python 3.5+
 
-Getting Started
-===============
+Installation
+============
 
-First, ensure that your Kafka instance is up and running:
+To install a stable version from PyPI_ (recommended):
 
 .. code-block:: bash
 
-    # This command is just an example
-    ~$ ./kafka-server-start.sh -daemon server.properties
-
+    ~$ pip install kq
 
-Let's say you want to run the following function asynchronously:
+To install the latest version directly from GitHub_:
 
-.. code-block:: python
+.. code-block:: bash
 
-    import time
+    ~$ pip install -e [email protected]:joowani/kq.git@master#egg=kq
 
-    def my_func(foo, bar, baz=None):
-        """This is a blocking function."""
-        time.sleep(10)
-        return foo, bar, baz
+You may need to use ``sudo`` depending on your environment.
 
+Getting Started
+===============
 
-Start a KQ worker:
+First, ensure that your Kafka instance is up and running:
 
 .. code-block:: bash
 
-    ~$ kq worker --verbose
-    [INFO] Starting Worker(topic=default) ...
-
+    ~$ ./kafka-server-start.sh -daemon server.properties
 
-Enqueue the function call as a job:
+Define your KQ worker module:
 
 .. code-block:: python
 
-    # Import the blocking function
-    from my_module import my_func
+    # my_worker.py
 
-    # Initialize a queue
-    from kq import Queue
-    q = Queue()
+    import logging
 
-    # Enqueue the function call
-    q.enqueue(my_func, 1, 2, baz=3)
+    from kafka import KafkaConsumer
+    from kq import Worker
 
+    # Set up logging.
+    formatter = logging.Formatter('[%(levelname)s] %(message)s')
+    stream_handler = logging.StreamHandler()
+    stream_handler.setFormatter(formatter)
+    logger = logging.getLogger('kq.worker')
+    logger.setLevel(logging.DEBUG)
+    logger.addHandler(stream_handler)
 
-Sit back and watch the worker process it in the background:
+    # Set up a Kafka consumer.
+    consumer = KafkaConsumer(
+        bootstrap_servers='127.0.0.1:9092',
+        group_id='group',
+        auto_offset_reset='latest'
+    )
 
-.. code-block:: bash
+    # Set up a worker.
+    worker = Worker(topic='topic', consumer=consumer)
+    worker.start()
 
-    ~$ kq worker --verbose
-    [INFO] Starting Worker(topic=default) ...
-    [INFO] Processing Record(topic=default, partition=5, offset=3) ...
-    [INFO] Running Job 1b92xle0: my_module.my_func(1, 2, baz=3) ...
-    [INFO] Job 1b92xle0 returned: (1, 2, 3)
+Start the worker:
 
+.. code-block:: bash
 
-Check out the full documentation_ for more details!
+    ~$ python my_worker.py
+    [INFO] Starting Worker(hosts=127.0.0.1:9092 topic=topic, group=group) ...
 
-.. _documentation: http://kq.readthedocs.io/en/master/
+Enqueue a function call:
 
+.. code-block:: python
 
-Installation
-============
+    import requests
 
-To install a stable version from PyPI_ (recommended):
+    from kafka import KafkaProducer
+    from kq import Queue
 
-.. code-block:: bash
+    # Set up a Kafka producer.
+    producer = KafkaProducer(bootstrap_servers='127.0.0.1:9092')
 
-    ~$ pip install kq
+    # Set up a queue.
+    queue = Queue(topic='topic', producer=producer)
 
+    # Enqueue a function call.
+    job = queue.enqueue(requests.get, 'https://www.google.com')
 
-To install the latest version directly from GitHub_:
+Sit back and watch the worker process it in the background:
 
 .. code-block:: bash
 
-    ~$ pip install -e [email protected]:joowani/kq.git@master#egg=kq
+    ~$ python my_worker.py
+    [INFO] Starting Worker(hosts=127.0.0.1:9092, topic=topic, group=group) ...
+    [INFO] Processing Message(topic=topic, partition=0, offset=0) ...
+    [INFO] Executing job c7bf2359: requests.api.get('https://www.google.com')
+    [INFO] Job c7bf2359 returned: <Response [200]>
 
-You may need to use ``sudo`` depending on your environment setup.
+**NEW in 2.0.0**: You can now specify the job timeout, message key and partition:
 
-.. _PyPI: https://pypi.python.org/pypi/kq
-.. _GitHub: https://github.com/joowani/kq
+.. code-block:: python
 
+    job = queue.using(timeout=5, key=b'foo', partition=0).enqueue(requests.get, 'https://www.google.com')
+
+Check out the full documentation_ for more information.
 
 Contributing
 ============
 
 Please have a look at this page_ before submitting a pull request. Thanks!
 
-.. _page:
-    http://kq.readthedocs.io/en/master/contributing.html
-
 
 Credits
 =======
 
-This project was inspired by RQ_ and built on top of kafka-python_.
+This project was inspired by RQ_.
 
-.. _RQ: https://github.com/nvie/rq
+.. _Apache Kafka: https://kafka.apache.org
 .. _kafka-python: https://github.com/dpkp/kafka-python
+.. _2.0.0: https://github.com/joowani/kq/releases/tag/2.0.0
+.. _releases: https://github.com/joowani/kq/releases
+.. _PyPI: https://pypi.python.org/pypi/kq
+.. _GitHub: https://github.com/joowani/kq
+.. _documentation: http://kq.readthedocs.io
+.. _page: http://kq.readthedocs.io/en/master/contributing.html
+.. _RQ: https://github.com/rq/rq