From 15a554678badf65a70690c2a1458f26fb3a23e0f Mon Sep 17 00:00:00 2001 From: William Glass Date: Wed, 13 Apr 2016 23:52:26 -0700 Subject: [PATCH] Fit for initial public consumption! --- .travis.yml | 18 + LICENSE | 202 +++++++++++ MANIFEST.in | 1 + README.rst | 85 +++++ docs/code/client.rst | 8 + docs/code/exceptions.rst | 6 + docs/code/helpers.rst | 6 + docs/code/modules/client.rst | 5 + docs/code/modules/connection.rst | 5 + docs/code/modules/exc.rst | 5 + docs/code/modules/features.rst | 5 + docs/code/modules/iterables.rst | 5 + docs/code/modules/protocol.acl.rst | 5 + docs/code/modules/protocol.auth.rst | 5 + docs/code/modules/protocol.check.rst | 5 + docs/code/modules/protocol.children.rst | 5 + docs/code/modules/protocol.close.rst | 5 + docs/code/modules/protocol.connect.rst | 5 + docs/code/modules/protocol.create.rst | 5 + docs/code/modules/protocol.data.rst | 5 + docs/code/modules/protocol.delete.rst | 5 + docs/code/modules/protocol.exists.rst | 5 + docs/code/modules/protocol.part.rst | 5 + docs/code/modules/protocol.ping.rst | 5 + docs/code/modules/protocol.primitives.rst | 5 + docs/code/modules/protocol.reconfig.rst | 5 + docs/code/modules/protocol.request.rst | 5 + docs/code/modules/protocol.response.rst | 5 + docs/code/modules/protocol.sasl.rst | 5 + docs/code/modules/protocol.stat.rst | 5 + docs/code/modules/protocol.transaction.rst | 5 + docs/code/modules/protocol.watches.rst | 5 + docs/code/modules/recipes.allocator.rst | 5 + docs/code/modules/recipes.barrier.rst | 5 + docs/code/modules/recipes.base_lock.rst | 5 + docs/code/modules/recipes.base_watcher.rst | 5 + .../code/modules/recipes.children_watcher.rst | 5 + docs/code/modules/recipes.counter.rst | 5 + docs/code/modules/recipes.data_watcher.rst | 5 + docs/code/modules/recipes.double_barrier.rst | 5 + docs/code/modules/recipes.election.rst | 5 + docs/code/modules/recipes.lease.rst | 5 + docs/code/modules/recipes.lock.rst | 5 + docs/code/modules/recipes.party.rst | 5 + docs/code/modules/recipes.proxy.rst | 5 + docs/code/modules/recipes.recipe.rst | 5 + docs/code/modules/recipes.sequential.rst | 5 + docs/code/modules/recipes.shared_lock.rst | 5 + docs/code/modules/recipes.tree_cache.rst | 5 + docs/code/modules/session.rst | 5 + docs/code/modules/states.rst | 5 + docs/code/modules/transaction.rst | 5 + docs/code/protocol.rst | 21 ++ docs/code/protocol_basics.rst | 9 + docs/code/recipe_basics.rst | 10 + docs/code/recipes.rst | 17 + docs/code/sessions.rst | 8 + docs/conf.py | 335 ++++++++++++++++++ docs/index.rst | 66 ++++ docs/releases.rst | 9 + docs/releases/0.8.0.rst | 4 + docs/requirements.txt | 4 + docs/source_docs.rst | 14 + docs/static/custom.css | 64 ++++ docs/templates/page.html | 5 + examples/__init__.py | 0 examples/allocator.py | 85 +++++ examples/barrier.py | 52 +++ examples/child_watcher.py | 42 +++ examples/counter.py | 71 ++++ examples/data_watcher.py | 38 ++ examples/double_barrier.py | 60 ++++ examples/election.py | 45 +++ examples/lease.py | 47 +++ examples/locking.py | 54 +++ examples/party.py | 51 +++ examples/run | 98 +++++ examples/runtime_config.py | 93 +++++ examples/shared_locking.py | 58 +++ examples/transactions.py | 123 +++++++ setup.cfg | 4 + setup.py | 61 ++++ tests/__init__.py | 0 tests/client_tests.py | 21 ++ tests/style_tests.py | 48 +++ tox.ini | 13 + zoonado/__init__.py | 8 + zoonado/client.py | 230 ++++++++++++ zoonado/connection.py | 236 ++++++++++++ zoonado/exc.py | 180 ++++++++++ zoonado/features.py | 12 + zoonado/iterables.py | 26 ++ zoonado/protocol/__init__.py | 89 +++++ zoonado/protocol/acl.py | 113 ++++++ zoonado/protocol/auth.py | 27 ++ zoonado/protocol/check.py | 22 ++ zoonado/protocol/children.py | 47 +++ zoonado/protocol/close.py | 22 ++ zoonado/protocol/connect.py | 27 ++ zoonado/protocol/create.py | 62 ++++ zoonado/protocol/data.py | 50 +++ zoonado/protocol/delete.py | 24 ++ zoonado/protocol/exists.py | 25 ++ zoonado/protocol/part.py | 100 ++++++ zoonado/protocol/ping.py | 22 ++ zoonado/protocol/primitives.py | 255 +++++++++++++ zoonado/protocol/reconfig.py | 27 ++ zoonado/protocol/request.py | 44 +++ zoonado/protocol/response.py | 39 ++ zoonado/protocol/sasl.py | 23 ++ zoonado/protocol/stat.py | 36 ++ zoonado/protocol/sync.py | 23 ++ zoonado/protocol/transaction.py | 109 ++++++ zoonado/protocol/watches.py | 88 +++++ zoonado/recipes/__init__.py | 2 + zoonado/recipes/allocator.py | 145 ++++++++ zoonado/recipes/barrier.py | 47 +++ zoonado/recipes/base_lock.py | 83 +++++ zoonado/recipes/base_watcher.py | 49 +++ zoonado/recipes/children_watcher.py | 14 + zoonado/recipes/counter.py | 90 +++++ zoonado/recipes/data_watcher.py | 14 + zoonado/recipes/double_barrier.py | 76 ++++ zoonado/recipes/election.py | 55 +++ zoonado/recipes/lease.py | 39 ++ zoonado/recipes/lock.py | 19 + zoonado/recipes/party.py | 46 +++ zoonado/recipes/proxy.py | 63 ++++ zoonado/recipes/recipe.py | 48 +++ zoonado/recipes/sequential.py | 88 +++++ zoonado/recipes/shared_lock.py | 32 ++ zoonado/recipes/tree_cache.py | 128 +++++++ zoonado/retry.py | 82 +++++ zoonado/session.py | 274 ++++++++++++++ zoonado/states.py | 66 ++++ zoonado/transaction.py | 87 +++++ 136 files changed, 5694 insertions(+) create mode 100644 .travis.yml create mode 100644 LICENSE create mode 100644 MANIFEST.in create mode 100644 README.rst create mode 100644 docs/code/client.rst create mode 100644 docs/code/exceptions.rst create mode 100644 docs/code/helpers.rst create mode 100644 docs/code/modules/client.rst create mode 100644 docs/code/modules/connection.rst create mode 100644 docs/code/modules/exc.rst create mode 100644 docs/code/modules/features.rst create mode 100644 docs/code/modules/iterables.rst create mode 100644 docs/code/modules/protocol.acl.rst create mode 100644 docs/code/modules/protocol.auth.rst create mode 100644 docs/code/modules/protocol.check.rst create mode 100644 docs/code/modules/protocol.children.rst create mode 100644 docs/code/modules/protocol.close.rst create mode 100644 docs/code/modules/protocol.connect.rst create mode 100644 docs/code/modules/protocol.create.rst create mode 100644 docs/code/modules/protocol.data.rst create mode 100644 docs/code/modules/protocol.delete.rst create mode 100644 docs/code/modules/protocol.exists.rst create mode 100644 docs/code/modules/protocol.part.rst create mode 100644 docs/code/modules/protocol.ping.rst create mode 100644 docs/code/modules/protocol.primitives.rst create mode 100644 docs/code/modules/protocol.reconfig.rst create mode 100644 docs/code/modules/protocol.request.rst create mode 100644 docs/code/modules/protocol.response.rst create mode 100644 docs/code/modules/protocol.sasl.rst create mode 100644 docs/code/modules/protocol.stat.rst create mode 100644 docs/code/modules/protocol.transaction.rst create mode 100644 docs/code/modules/protocol.watches.rst create mode 100644 docs/code/modules/recipes.allocator.rst create mode 100644 docs/code/modules/recipes.barrier.rst create mode 100644 docs/code/modules/recipes.base_lock.rst create mode 100644 docs/code/modules/recipes.base_watcher.rst create mode 100644 docs/code/modules/recipes.children_watcher.rst create mode 100644 docs/code/modules/recipes.counter.rst create mode 100644 docs/code/modules/recipes.data_watcher.rst create mode 100644 docs/code/modules/recipes.double_barrier.rst create mode 100644 docs/code/modules/recipes.election.rst create mode 100644 docs/code/modules/recipes.lease.rst create mode 100644 docs/code/modules/recipes.lock.rst create mode 100644 docs/code/modules/recipes.party.rst create mode 100644 docs/code/modules/recipes.proxy.rst create mode 100644 docs/code/modules/recipes.recipe.rst create mode 100644 docs/code/modules/recipes.sequential.rst create mode 100644 docs/code/modules/recipes.shared_lock.rst create mode 100644 docs/code/modules/recipes.tree_cache.rst create mode 100644 docs/code/modules/session.rst create mode 100644 docs/code/modules/states.rst create mode 100644 docs/code/modules/transaction.rst create mode 100644 docs/code/protocol.rst create mode 100644 docs/code/protocol_basics.rst create mode 100644 docs/code/recipe_basics.rst create mode 100644 docs/code/recipes.rst create mode 100644 docs/code/sessions.rst create mode 100644 docs/conf.py create mode 100644 docs/index.rst create mode 100644 docs/releases.rst create mode 100644 docs/releases/0.8.0.rst create mode 100644 docs/requirements.txt create mode 100644 docs/source_docs.rst create mode 100644 docs/static/custom.css create mode 100644 docs/templates/page.html create mode 100644 examples/__init__.py create mode 100644 examples/allocator.py create mode 100644 examples/barrier.py create mode 100644 examples/child_watcher.py create mode 100644 examples/counter.py create mode 100644 examples/data_watcher.py create mode 100644 examples/double_barrier.py create mode 100644 examples/election.py create mode 100644 examples/lease.py create mode 100644 examples/locking.py create mode 100644 examples/party.py create mode 100755 examples/run create mode 100644 examples/runtime_config.py create mode 100644 examples/shared_locking.py create mode 100644 examples/transactions.py create mode 100644 setup.cfg create mode 100644 setup.py create mode 100644 tests/__init__.py create mode 100644 tests/client_tests.py create mode 100644 tests/style_tests.py create mode 100644 tox.ini create mode 100644 zoonado/__init__.py create mode 100644 zoonado/client.py create mode 100644 zoonado/connection.py create mode 100644 zoonado/exc.py create mode 100644 zoonado/features.py create mode 100644 zoonado/iterables.py create mode 100644 zoonado/protocol/__init__.py create mode 100644 zoonado/protocol/acl.py create mode 100644 zoonado/protocol/auth.py create mode 100644 zoonado/protocol/check.py create mode 100644 zoonado/protocol/children.py create mode 100644 zoonado/protocol/close.py create mode 100644 zoonado/protocol/connect.py create mode 100644 zoonado/protocol/create.py create mode 100644 zoonado/protocol/data.py create mode 100644 zoonado/protocol/delete.py create mode 100644 zoonado/protocol/exists.py create mode 100644 zoonado/protocol/part.py create mode 100644 zoonado/protocol/ping.py create mode 100644 zoonado/protocol/primitives.py create mode 100644 zoonado/protocol/reconfig.py create mode 100644 zoonado/protocol/request.py create mode 100644 zoonado/protocol/response.py create mode 100644 zoonado/protocol/sasl.py create mode 100644 zoonado/protocol/stat.py create mode 100644 zoonado/protocol/sync.py create mode 100644 zoonado/protocol/transaction.py create mode 100644 zoonado/protocol/watches.py create mode 100644 zoonado/recipes/__init__.py create mode 100644 zoonado/recipes/allocator.py create mode 100644 zoonado/recipes/barrier.py create mode 100644 zoonado/recipes/base_lock.py create mode 100644 zoonado/recipes/base_watcher.py create mode 100644 zoonado/recipes/children_watcher.py create mode 100644 zoonado/recipes/counter.py create mode 100644 zoonado/recipes/data_watcher.py create mode 100644 zoonado/recipes/double_barrier.py create mode 100644 zoonado/recipes/election.py create mode 100644 zoonado/recipes/lease.py create mode 100644 zoonado/recipes/lock.py create mode 100644 zoonado/recipes/party.py create mode 100644 zoonado/recipes/proxy.py create mode 100644 zoonado/recipes/recipe.py create mode 100644 zoonado/recipes/sequential.py create mode 100644 zoonado/recipes/shared_lock.py create mode 100644 zoonado/recipes/tree_cache.py create mode 100644 zoonado/retry.py create mode 100644 zoonado/session.py create mode 100644 zoonado/states.py create mode 100644 zoonado/transaction.py diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..b6437ed --- /dev/null +++ b/.travis.yml @@ -0,0 +1,18 @@ +language: python +envs: + - TOXENV=py27 + - TOXENV=py34 + - TOXENV=pypy +before_install: + - pip install codecov +install: pip install tox +script: tox +after_success: + - codecov +deploy: + - provider: pypi + user: wglass + password: + secure: tc7INI/VL+YIQU22VD+XogrN++48mA/HHxsnVNJooDypOfk6kSr0HT0VmuizXY+eL2aS7WbI3RPQI/kDJjxpz8XnuW7JIvAOJ+EGzVXSDNprdg1Rtlg3GTxRgtTaJteiXge7DOrHuewpJVq8JifofsFW76uz3yvoFq/GaXA1wLgUPqyUQ7VqSRCDbzoBtmL7EBlHWwprXCPcgYAEP6PCKHBpmuXo3pIdPoo5rRme3Rd4MP2SRhoI7QheKhvP0p/EDzfFRt/3qfNeXZ2QT1QEdXgtAYF3uuJRzScgGIRI0TwRLCEqkqaWhFD/0g0YQwVvb+diQILv15zzdz4kIMA2xpXF1o9RsQyQFVAsRisAjJo2uV7oC7JtvqMcBzUq+S39Dl6KRKQdbcpFn739UauwmIgb27OqoPrkOiFne0I07Mbv6KA3aDv8V60YRTLMWaO0WnKEYK+enI4z2gbgmCpQ7zLQ8h2MJpHCSLraUlyJ0gf4umPBxNasGN1IVIX/8XH8zaweGG1DbVmiG8WKdgNHkZ0ljHDgK6YRTrTFkrYE7BU1UVlI+4+KQWiFwb9kzd5UMUxDCBazw5v+9/nWagquME0G035k9eO7vwhGkFvcAOS9Yk5h5H5DNVvk2+Kj62v7yDFYDRufZROB5RF+lmWENKMfv9/nfK1WjsaKFgCR1L8= + on: + tags: true diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..4b21c50 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2016 William Glass + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..8dce522 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include LICENSE README.rst diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..58518b6 --- /dev/null +++ b/README.rst @@ -0,0 +1,85 @@ +======================================= +Zoonado: Async Tornado Zookeeper Client +======================================= + +.. image:: + https://img.shields.io/pypi/v/zoonado.svg + :target: http://pypi.python.org/pypi/zoonado + :alt: Python Package Version +.. image:: + https://readthedocs.org/projects/zoonado/badge/?version=latest + :alt: Documentation Status + :target: http://zoonado.readthedocs.org/en/latest/ +.. image:: + https://travis-ci.org/wglass/zoonado.svg?branch=master + :alt: Build Status + :target: https://travis-ci.org/wglass/zoonado +.. image:: + https://landscape.io/github/wglass/zoonado/master/landscape.svg?style=flat + :alt: Code Health + :target: https://landscape.io/github/wglass/zoonado/master +.. image:: + https://codecov.io/github/wglass/zoonado/coverage.svg?branch=master + :alt: Codecov.io + :target: https://codecov.io/github/wglass/zoonado?branch=master + +.. + + Zoonado is a Zookeeper_ python client using Tornado_ to achieve async I/O. + + +.. contents:: :local: + + +Installation +~~~~~~~~~~~~ + +Zoonado is available via PyPI_, installation is as easy as:: + + pip install zoonado + + +Quick Example +~~~~~~~~~~~~~ + +:: + + from tornado import gen + from zoonado import Zoonado + + @gen.coroutine + def run(): + zk = Zoonado("zk01,zk02,zk03", chroot="/shared/namespace") + + yield zk.start() + + yield zk.create("/foo/bar", data="bazz", ephemeral=True) + + yield zk.set_data("/foo/bar", "bwee") + + yield zk.close() + + +Development +~~~~~~~~~~~ + +The code is hosted on GitHub_ + + +To file a bug or possible enhancement see the `Issue Tracker`_, also found +on GitHub. + + +License +~~~~~~~ + +Lighthouse is licensed under the terms of the Apache license (2.0). See the +LICENSE_ file for more details. + + +.. _Zookeeper: https://zookeeper.apache.org +.. _Tornado: http://tornadoweb.org +.. _PyPI: https://pypi.python.org/pypi/zoonado +.. _GitHub: https://github.com/wglass/zoonado +.. _`Issue Tracker`: https://github.com/wglass/zoonado/issues +.. _LICENSE: https://github.com/wglass/zoonado/blob/master/LICENSE diff --git a/docs/code/client.rst b/docs/code/client.rst new file mode 100644 index 0000000..3c2b70b --- /dev/null +++ b/docs/code/client.rst @@ -0,0 +1,8 @@ +Client +====== + +.. toctree:: + + modules/client + modules/features + modules/transaction diff --git a/docs/code/exceptions.rst b/docs/code/exceptions.rst new file mode 100644 index 0000000..b35e66b --- /dev/null +++ b/docs/code/exceptions.rst @@ -0,0 +1,6 @@ +Exceptions +========== + +.. toctree:: + + modules/exc diff --git a/docs/code/helpers.rst b/docs/code/helpers.rst new file mode 100644 index 0000000..560e1af --- /dev/null +++ b/docs/code/helpers.rst @@ -0,0 +1,6 @@ +Helper Modules +============== + +.. toctree:: + + modules/iterables diff --git a/docs/code/modules/client.rst b/docs/code/modules/client.rst new file mode 100644 index 0000000..acab1a3 --- /dev/null +++ b/docs/code/modules/client.rst @@ -0,0 +1,5 @@ +``zoonado.client`` +================== + +.. automodule:: zoonado.client + :members: diff --git a/docs/code/modules/connection.rst b/docs/code/modules/connection.rst new file mode 100644 index 0000000..55ab8a9 --- /dev/null +++ b/docs/code/modules/connection.rst @@ -0,0 +1,5 @@ +``zoonado.connection`` +====================== + +.. automodule:: zoonado.connection + :members: diff --git a/docs/code/modules/exc.rst b/docs/code/modules/exc.rst new file mode 100644 index 0000000..b026a98 --- /dev/null +++ b/docs/code/modules/exc.rst @@ -0,0 +1,5 @@ +``zoonado.exc`` +=============== + +.. automodule:: zoonado.exc + :members: diff --git a/docs/code/modules/features.rst b/docs/code/modules/features.rst new file mode 100644 index 0000000..9ca474c --- /dev/null +++ b/docs/code/modules/features.rst @@ -0,0 +1,5 @@ +``zoonado.features`` +==================== + +.. automodule:: zoonado.features + :members: diff --git a/docs/code/modules/iterables.rst b/docs/code/modules/iterables.rst new file mode 100644 index 0000000..7d62f60 --- /dev/null +++ b/docs/code/modules/iterables.rst @@ -0,0 +1,5 @@ +``zoonado.iterables`` +===================== + +.. automodule:: zoonado.iterables + :members: diff --git a/docs/code/modules/protocol.acl.rst b/docs/code/modules/protocol.acl.rst new file mode 100644 index 0000000..5467835 --- /dev/null +++ b/docs/code/modules/protocol.acl.rst @@ -0,0 +1,5 @@ +``zoonado.protocol.acl`` +======================== + +.. automodule:: zoonado.protocol.acl + :members: diff --git a/docs/code/modules/protocol.auth.rst b/docs/code/modules/protocol.auth.rst new file mode 100644 index 0000000..802693a --- /dev/null +++ b/docs/code/modules/protocol.auth.rst @@ -0,0 +1,5 @@ +``zoonado.protocol.auth`` +========================= + +.. automodule:: zoonado.protocol.auth + :members: diff --git a/docs/code/modules/protocol.check.rst b/docs/code/modules/protocol.check.rst new file mode 100644 index 0000000..df3699c --- /dev/null +++ b/docs/code/modules/protocol.check.rst @@ -0,0 +1,5 @@ +``zoonado.protocol.check`` +========================== + +.. automodule:: zoonado.protocol.check + :members: diff --git a/docs/code/modules/protocol.children.rst b/docs/code/modules/protocol.children.rst new file mode 100644 index 0000000..a524825 --- /dev/null +++ b/docs/code/modules/protocol.children.rst @@ -0,0 +1,5 @@ +``zoonado.protocol.children`` +============================= + +.. automodule:: zoonado.protocol.children + :members: diff --git a/docs/code/modules/protocol.close.rst b/docs/code/modules/protocol.close.rst new file mode 100644 index 0000000..b44ab89 --- /dev/null +++ b/docs/code/modules/protocol.close.rst @@ -0,0 +1,5 @@ +``zoonado.protocol.close`` +========================== + +.. automodule:: zoonado.protocol.close + :members: diff --git a/docs/code/modules/protocol.connect.rst b/docs/code/modules/protocol.connect.rst new file mode 100644 index 0000000..d9c3191 --- /dev/null +++ b/docs/code/modules/protocol.connect.rst @@ -0,0 +1,5 @@ +``zoonado.protocol.connect`` +============================ + +.. automodule:: zoonado.protocol.connect + :members: diff --git a/docs/code/modules/protocol.create.rst b/docs/code/modules/protocol.create.rst new file mode 100644 index 0000000..b6db5d0 --- /dev/null +++ b/docs/code/modules/protocol.create.rst @@ -0,0 +1,5 @@ +``zoonado.protocol.create`` +=========================== + +.. automodule:: zoonado.protocol.create + :members: diff --git a/docs/code/modules/protocol.data.rst b/docs/code/modules/protocol.data.rst new file mode 100644 index 0000000..fbd13bf --- /dev/null +++ b/docs/code/modules/protocol.data.rst @@ -0,0 +1,5 @@ +``zoonado.protocol.data`` +========================= + +.. automodule:: zoonado.protocol.data + :members: diff --git a/docs/code/modules/protocol.delete.rst b/docs/code/modules/protocol.delete.rst new file mode 100644 index 0000000..db3bbfd --- /dev/null +++ b/docs/code/modules/protocol.delete.rst @@ -0,0 +1,5 @@ +``zoonado.protocol.delete`` +=========================== + +.. automodule:: zoonado.protocol.delete + :members: diff --git a/docs/code/modules/protocol.exists.rst b/docs/code/modules/protocol.exists.rst new file mode 100644 index 0000000..b3533bc --- /dev/null +++ b/docs/code/modules/protocol.exists.rst @@ -0,0 +1,5 @@ +``zoonado.protocol.exists`` +=========================== + +.. automodule:: zoonado.protocol.exists + :members: diff --git a/docs/code/modules/protocol.part.rst b/docs/code/modules/protocol.part.rst new file mode 100644 index 0000000..4a60092 --- /dev/null +++ b/docs/code/modules/protocol.part.rst @@ -0,0 +1,5 @@ +``zoonado.protocol.part`` +========================= + +.. automodule:: zoonado.protocol.part + :members: diff --git a/docs/code/modules/protocol.ping.rst b/docs/code/modules/protocol.ping.rst new file mode 100644 index 0000000..372eef2 --- /dev/null +++ b/docs/code/modules/protocol.ping.rst @@ -0,0 +1,5 @@ +``zoonado.protocol.ping`` +========================= + +.. automodule:: zoonado.protocol.ping + :members: diff --git a/docs/code/modules/protocol.primitives.rst b/docs/code/modules/protocol.primitives.rst new file mode 100644 index 0000000..230ddc4 --- /dev/null +++ b/docs/code/modules/protocol.primitives.rst @@ -0,0 +1,5 @@ +``zoonado.protocol.primitives`` +=============================== + +.. automodule:: zoonado.protocol.primitives + :members: diff --git a/docs/code/modules/protocol.reconfig.rst b/docs/code/modules/protocol.reconfig.rst new file mode 100644 index 0000000..f70d720 --- /dev/null +++ b/docs/code/modules/protocol.reconfig.rst @@ -0,0 +1,5 @@ +``zoonado.protocol.reconfig`` +============================= + +.. automodule:: zoonado.protocol.reconfig + :members: diff --git a/docs/code/modules/protocol.request.rst b/docs/code/modules/protocol.request.rst new file mode 100644 index 0000000..6c2df4a --- /dev/null +++ b/docs/code/modules/protocol.request.rst @@ -0,0 +1,5 @@ +``zoonado.protocol.request`` +============================ + +.. automodule:: zoonado.protocol.request + :members: diff --git a/docs/code/modules/protocol.response.rst b/docs/code/modules/protocol.response.rst new file mode 100644 index 0000000..ab349e5 --- /dev/null +++ b/docs/code/modules/protocol.response.rst @@ -0,0 +1,5 @@ +``zoonado.protocol.response`` +============================= + +.. automodule:: zoonado.protocol.response + :members: diff --git a/docs/code/modules/protocol.sasl.rst b/docs/code/modules/protocol.sasl.rst new file mode 100644 index 0000000..1466f8d --- /dev/null +++ b/docs/code/modules/protocol.sasl.rst @@ -0,0 +1,5 @@ +``zoonado.protocol.sasl`` +========================= + +.. automodule:: zoonado.protocol.sasl + :members: diff --git a/docs/code/modules/protocol.stat.rst b/docs/code/modules/protocol.stat.rst new file mode 100644 index 0000000..5bdb8d5 --- /dev/null +++ b/docs/code/modules/protocol.stat.rst @@ -0,0 +1,5 @@ +``zoonado.protocol.stat`` +========================= + +.. automodule:: zoonado.protocol.stat + :members: diff --git a/docs/code/modules/protocol.transaction.rst b/docs/code/modules/protocol.transaction.rst new file mode 100644 index 0000000..ee826ab --- /dev/null +++ b/docs/code/modules/protocol.transaction.rst @@ -0,0 +1,5 @@ +``zoonado.protocol.transaction`` +================================ + +.. automodule:: zoonado.protocol.transaction + :members: diff --git a/docs/code/modules/protocol.watches.rst b/docs/code/modules/protocol.watches.rst new file mode 100644 index 0000000..d01bce9 --- /dev/null +++ b/docs/code/modules/protocol.watches.rst @@ -0,0 +1,5 @@ +``zoonado.protocol.watches`` +============================ + +.. automodule:: zoonado.protocol.watches + :members: diff --git a/docs/code/modules/recipes.allocator.rst b/docs/code/modules/recipes.allocator.rst new file mode 100644 index 0000000..8362e71 --- /dev/null +++ b/docs/code/modules/recipes.allocator.rst @@ -0,0 +1,5 @@ +``zoonado.recipes.allocator`` +============================= + +.. automodule:: zoonado.recipes.allocator + :members: diff --git a/docs/code/modules/recipes.barrier.rst b/docs/code/modules/recipes.barrier.rst new file mode 100644 index 0000000..0bc8a17 --- /dev/null +++ b/docs/code/modules/recipes.barrier.rst @@ -0,0 +1,5 @@ +``zoonado.recipes.barrier`` +=========================== + +.. automodule:: zoonado.recipes.barrier + :members: diff --git a/docs/code/modules/recipes.base_lock.rst b/docs/code/modules/recipes.base_lock.rst new file mode 100644 index 0000000..3474925 --- /dev/null +++ b/docs/code/modules/recipes.base_lock.rst @@ -0,0 +1,5 @@ +``zoonado.recipes.base_lock`` +============================= + +.. automodule:: zoonado.recipes.base_lock + :members: diff --git a/docs/code/modules/recipes.base_watcher.rst b/docs/code/modules/recipes.base_watcher.rst new file mode 100644 index 0000000..c9db09d --- /dev/null +++ b/docs/code/modules/recipes.base_watcher.rst @@ -0,0 +1,5 @@ +``zoonado.recipes.base_watcher`` +================================ + +.. automodule:: zoonado.recipes.base_watcher + :members: diff --git a/docs/code/modules/recipes.children_watcher.rst b/docs/code/modules/recipes.children_watcher.rst new file mode 100644 index 0000000..250346c --- /dev/null +++ b/docs/code/modules/recipes.children_watcher.rst @@ -0,0 +1,5 @@ +``zoonado.recipes.children_watcher`` +==================================== + +.. automodule:: zoonado.recipes.children_watcher + :members: diff --git a/docs/code/modules/recipes.counter.rst b/docs/code/modules/recipes.counter.rst new file mode 100644 index 0000000..1bbb0d5 --- /dev/null +++ b/docs/code/modules/recipes.counter.rst @@ -0,0 +1,5 @@ +``zoonado.recipes.counter`` +=========================== + +.. automodule:: zoonado.recipes.counter + :members: diff --git a/docs/code/modules/recipes.data_watcher.rst b/docs/code/modules/recipes.data_watcher.rst new file mode 100644 index 0000000..8f5aec7 --- /dev/null +++ b/docs/code/modules/recipes.data_watcher.rst @@ -0,0 +1,5 @@ +``zoonado.recipes.data_watcher`` +================================ + +.. automodule:: zoonado.recipes.data_watcher + :members: diff --git a/docs/code/modules/recipes.double_barrier.rst b/docs/code/modules/recipes.double_barrier.rst new file mode 100644 index 0000000..95d4e94 --- /dev/null +++ b/docs/code/modules/recipes.double_barrier.rst @@ -0,0 +1,5 @@ +``zoonado.recipes.double_barrier`` +================================== + +.. automodule:: zoonado.recipes.double_barrier + :members: diff --git a/docs/code/modules/recipes.election.rst b/docs/code/modules/recipes.election.rst new file mode 100644 index 0000000..68bf2d6 --- /dev/null +++ b/docs/code/modules/recipes.election.rst @@ -0,0 +1,5 @@ +``zoonado.recipes.election`` +============================ + +.. automodule:: zoonado.recipes.election + :members: diff --git a/docs/code/modules/recipes.lease.rst b/docs/code/modules/recipes.lease.rst new file mode 100644 index 0000000..a8d14a0 --- /dev/null +++ b/docs/code/modules/recipes.lease.rst @@ -0,0 +1,5 @@ +``zoonado.recipes.lease`` +========================= + +.. automodule:: zoonado.recipes.lease + :members: diff --git a/docs/code/modules/recipes.lock.rst b/docs/code/modules/recipes.lock.rst new file mode 100644 index 0000000..54dfc62 --- /dev/null +++ b/docs/code/modules/recipes.lock.rst @@ -0,0 +1,5 @@ +``zoonado.recipes.lock`` +======================== + +.. automodule:: zoonado.recipes.lock + :members: diff --git a/docs/code/modules/recipes.party.rst b/docs/code/modules/recipes.party.rst new file mode 100644 index 0000000..6471380 --- /dev/null +++ b/docs/code/modules/recipes.party.rst @@ -0,0 +1,5 @@ +``zoonado.recipes.party`` +========================= + +.. automodule:: zoonado.recipes.party + :members: diff --git a/docs/code/modules/recipes.proxy.rst b/docs/code/modules/recipes.proxy.rst new file mode 100644 index 0000000..05769f7 --- /dev/null +++ b/docs/code/modules/recipes.proxy.rst @@ -0,0 +1,5 @@ +``zoonado.recipes.proxy`` +========================= + +.. automodule:: zoonado.recipes.proxy + :members: diff --git a/docs/code/modules/recipes.recipe.rst b/docs/code/modules/recipes.recipe.rst new file mode 100644 index 0000000..83166ca --- /dev/null +++ b/docs/code/modules/recipes.recipe.rst @@ -0,0 +1,5 @@ +``zoonado.recipes.recipe`` +========================== + +.. automodule:: zoonado.recipes.recipe + :members: diff --git a/docs/code/modules/recipes.sequential.rst b/docs/code/modules/recipes.sequential.rst new file mode 100644 index 0000000..4dcc2ea --- /dev/null +++ b/docs/code/modules/recipes.sequential.rst @@ -0,0 +1,5 @@ +``zoonado.recipes.sequential`` +============================== + +.. automodule:: zoonado.recipes.sequential + :members: diff --git a/docs/code/modules/recipes.shared_lock.rst b/docs/code/modules/recipes.shared_lock.rst new file mode 100644 index 0000000..4c88104 --- /dev/null +++ b/docs/code/modules/recipes.shared_lock.rst @@ -0,0 +1,5 @@ +``zoonado.recipes.shared_lock`` +=============================== + +.. automodule:: zoonado.recipes.shared_lock + :members: diff --git a/docs/code/modules/recipes.tree_cache.rst b/docs/code/modules/recipes.tree_cache.rst new file mode 100644 index 0000000..6efcfdb --- /dev/null +++ b/docs/code/modules/recipes.tree_cache.rst @@ -0,0 +1,5 @@ +``zoonado.recipes.tree_cache`` +============================== + +.. automodule:: zoonado.recipes.tree_cache + :members: diff --git a/docs/code/modules/session.rst b/docs/code/modules/session.rst new file mode 100644 index 0000000..f58d1e2 --- /dev/null +++ b/docs/code/modules/session.rst @@ -0,0 +1,5 @@ +``zoonado.session`` +=================== + +.. automodule:: zoonado.session + :members: diff --git a/docs/code/modules/states.rst b/docs/code/modules/states.rst new file mode 100644 index 0000000..e704021 --- /dev/null +++ b/docs/code/modules/states.rst @@ -0,0 +1,5 @@ +``zoonado.states`` +================== + +.. automodule:: zoonado.states + :members: diff --git a/docs/code/modules/transaction.rst b/docs/code/modules/transaction.rst new file mode 100644 index 0000000..5817463 --- /dev/null +++ b/docs/code/modules/transaction.rst @@ -0,0 +1,5 @@ +``zoonado.transaction`` +======================= + +.. automodule:: zoonado.transaction + :members: diff --git a/docs/code/protocol.rst b/docs/code/protocol.rst new file mode 100644 index 0000000..7f9b521 --- /dev/null +++ b/docs/code/protocol.rst @@ -0,0 +1,21 @@ +The Protocol +============ + +.. toctree:: + + modules/protocol.connect + modules/protocol.close + modules/protocol.ping + modules/protocol.exists + modules/protocol.stat + modules/protocol.create + modules/protocol.data + modules/protocol.children + modules/protocol.delete + modules/protocol.watches + modules/protocol.transaction + modules/protocol.check + modules/protocol.acl + modules/protocol.sasl + modules/protocol.auth + modules/protocol.reconfig diff --git a/docs/code/protocol_basics.rst b/docs/code/protocol_basics.rst new file mode 100644 index 0000000..06a6398 --- /dev/null +++ b/docs/code/protocol_basics.rst @@ -0,0 +1,9 @@ +Protocol Basics +=============== + +.. toctree:: + + modules/protocol.primitives + modules/protocol.part + modules/protocol.request + modules/protocol.response diff --git a/docs/code/recipe_basics.rst b/docs/code/recipe_basics.rst new file mode 100644 index 0000000..3b77656 --- /dev/null +++ b/docs/code/recipe_basics.rst @@ -0,0 +1,10 @@ +Recipe Basics +============= + +.. toctree:: + + modules/recipes.recipe + modules/recipes.sequential + modules/recipes.proxy + modules/recipes.base_watcher + modules/recipes.base_lock diff --git a/docs/code/recipes.rst b/docs/code/recipes.rst new file mode 100644 index 0000000..e1fe528 --- /dev/null +++ b/docs/code/recipes.rst @@ -0,0 +1,17 @@ +Recipes +======= + +.. toctree:: + + modules/recipes.lock + modules/recipes.shared_lock + modules/recipes.data_watcher + modules/recipes.children_watcher + modules/recipes.election + modules/recipes.counter + modules/recipes.barrier + modules/recipes.double_barrier + modules/recipes.lease + modules/recipes.party + modules/recipes.tree_cache + modules/recipes.allocator diff --git a/docs/code/sessions.rst b/docs/code/sessions.rst new file mode 100644 index 0000000..f2b2f06 --- /dev/null +++ b/docs/code/sessions.rst @@ -0,0 +1,8 @@ +Sessions +======== + +.. toctree:: + + modules/session + modules/connection + modules/states diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..5bdf066 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,335 @@ +# -*- coding: utf-8 -*- +# +# zoonado documentation build configuration file, created by +# sphinx-quickstart on Wed Dec 23 16:22:07 2015. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys +import os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.insert(0, os.path.abspath('..')) + +import sphinx_bootstrap_theme +import zoonado + +on_rtd = os.environ.get('READTHEDOCS', None) == 'True' + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.doctest', + 'sphinx.ext.coverage', + 'sphinx.ext.viewcode', + 'sphinx.ext.extlinks', +] +if not on_rtd: + extensions.append("sphinxcontrib.spelling") + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'zoonado' +copyright = u'2016, William Glass' +author = u'William Glass' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# + +version = release = zoonado.__version__ + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + + +autodoc_member_order = "bysource" +autoclass_content = "both" +autodoc_docstring_signature = False + +coverage_skip_undoc_in_source = True +coverage_ignore_modules = [] + +spelling_word_list_filename = "spelling_wordlist.txt" + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = [] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +default_role = "py:obj" + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'trac' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +#keep_warnings = False + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ---------------------------------------------- + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['static'] + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme_path = sphinx_bootstrap_theme.get_html_theme_path() +html_theme = 'bootstrap' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. + +html_theme_options = { + "navbar_links": [ + # ("Basics", "basics"), + # ("Recipes", "recipes"), + # ("Security", "security"), + # ("The Session", "sessions"), + # ("The Protocol", "protocol"), + ("Release Notes", "releases"), + ("Source Docs", "source_docs"), + ], + 'navbar_class': "navbar", + "navbar_site_name": "Site", + "globaltoc_depth": 2, + "globaltoc_includehidden": False, + "navbar_sidebarrel": False, + "navbar_pagenav": True, + "source_link_position": None, + "bootswatch_theme": "paper", + "bootstrap_version": "3", +} + +# Add any paths that contain custom themes here, relative to this directory. + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Custom sidebar templates, maps document names to template names. +# html_sidebars = { +# '**': [ +# 'about.html', 'navigation.html', 'searchbox.html', 'donate.html', +# ] +# } + +extlinks = { + "current_tarball": ( + "https://pypi.python.org/packages/source/z/zoonado/" + + "zoonado-%s.tar.g%%s" % version, + "zoonado-%s.tar.g" % version + ) +} + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +#html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Language to be used for generating the HTML full-text search index. +# Sphinx supports the following languages: +# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' +# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' +#html_search_language = 'en' + +# A dictionary with options for the search language support, empty by default. +# Now only 'ja' uses this config value +#html_search_options = {'type': 'default'} + +# The name of a javascript file (relative to the configuration directory) that +# implements a search results scorer. If empty, the default will be used. +#html_search_scorer = 'scorer.js' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'zoonadodoc' + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', + +# The font size ('10pt', '11pt' or '12pt'). +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +#'preamble': '', + +# Latex figure (float) alignment +#'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'zk-tornado.tex', u'Zoonado Documentation', + u'William Glass', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'zoonado', u'Zoonado Documentation', + [author], 1) +] + +# If true, show URL addresses after external links. +#man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'zoonado', u'Zoonado Documentation', + author, 'zoonado', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +#texinfo_no_detailmenu = False diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..b446184 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,66 @@ +.. title:: Zoonado: Async Tornado Zookeeper Client + +Async Tornado Zookeeper Client +============================== + + +.. + + Zoonado is a Zookeeper_ python client using Tornado_ to achieve async I/O. + + +Installation +------------ + +Pip +~~~ + +Zoonado is available via PyPI_, installation is as easy as:: + + pip install zoonado + + +Manual +~~~~~~ + +To install manually, first download and unzip the :current_tarball:`z`, then: + +.. parsed-literal:: + + tar -zxvf zoonado-|version|.tar.gz + cd zoonado-|version| + python setup.py install + + +Development +----------- + +The code is hosted on GitHub_ + + +To file a bug or possible enhancement see the `Issue Tracker`_, also found +on GitHub. + + +License +------- + +Lighthouse is licensed under the terms of the Apache license (2.0). See the +LICENSE_ file for more details. + + +.. _Zookeeper: https://zookeeper.apache.org +.. _Tornado: http://tornadoweb.org +.. _PyPI: https://pypi.pyton.org/pypi/zoonado +.. _GitHub: https://github.com/wglass/zoonado +.. _`Issue Tracker`: https://github.com/wglass/zoonado/issues +.. _LICENSE: https://github.com/wglass/zoonado/blob/master/LICENSE + + +.. toctree:: + :hidden: + :titlesonly: + :maxdepth: 2 + + source_docs + releases diff --git a/docs/releases.rst b/docs/releases.rst new file mode 100644 index 0000000..75d7805 --- /dev/null +++ b/docs/releases.rst @@ -0,0 +1,9 @@ +================ +Release Notes +================ + +.. toctree:: + :maxdepth: 2 + :glob: + + releases/* diff --git a/docs/releases/0.8.0.rst b/docs/releases/0.8.0.rst new file mode 100644 index 0000000..3fe9fed --- /dev/null +++ b/docs/releases/0.8.0.rst @@ -0,0 +1,4 @@ +0.8.0 +~~~~~ + +* Initial public release diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..a46ef9e --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,4 @@ +Sphinx +sphinx-bootstrap-theme +sphinxcontrib-spelling +pyenchant diff --git a/docs/source_docs.rst b/docs/source_docs.rst new file mode 100644 index 0000000..bbc9c08 --- /dev/null +++ b/docs/source_docs.rst @@ -0,0 +1,14 @@ +Source Docs +=========== + +.. toctree:: + :titlesonly: + + code/client + code/sessions + code/recipe_basics + code/recipes + code/protocol_basics + code/protocol + code/exceptions + code/helpers diff --git a/docs/static/custom.css b/docs/static/custom.css new file mode 100644 index 0000000..be7ef04 --- /dev/null +++ b/docs/static/custom.css @@ -0,0 +1,64 @@ +.navbar-default { + background-color: #476c9b; +} +.navbar-default a { + text-decoration: none; +} +.navbar-default .navbar-brand { + color: #ffffff; +} +.navbar-default .navbar-brand:hover { + color: #d6e3f8; +} +.navbar-default .navbar-nav>li>a { + color: #ffffff; +} +.navbar-default .navbar-nav>li>a:hover { + color: #d6e3f8; +} +.navbar-form .form-control { + color: #ffffff; +} +.navbar-form .form-control:focus { + box-shadow: inset 0 -2px 0 #468c98; +} +.alert-info { + background-color: #476c9b; +} +.alert-info code { + color: #ffffff; + background-color: #476c9b; +} +.alert-warning { + background-color: #984447; +} +.alert-warning code { + color: #ffffff; + background-color: #984447; +} +.alert { + color: #ffffff; +} +.alert a:not(.close) { + color: #ffffff; + font-weight: italic; + text-decoration: underline; +} +.alert-warning a:not(.close) { + color: #ffffff; + font-style: italic; +} +a { + color: #476c9b; + font-weight: bold; +} +a:hover { + color: #476c9b; + text-decoration: underline; +} +code { + color: #92140c; +} +dt:target, .highlighted { + background-color: #c1b4ae; +} diff --git a/docs/templates/page.html b/docs/templates/page.html new file mode 100644 index 0000000..c65ef34 --- /dev/null +++ b/docs/templates/page.html @@ -0,0 +1,5 @@ +{# Import the theme's layout. #} +{% extends "!page.html" %} + +{# Custom CSS overrides #} +{% set bootswatch_css_custom = ['_static/custom.css'] %} diff --git a/examples/__init__.py b/examples/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/examples/allocator.py b/examples/allocator.py new file mode 100644 index 0000000..8b61ff5 --- /dev/null +++ b/examples/allocator.py @@ -0,0 +1,85 @@ +import collections +import itertools +import logging +import random + +from tornado import gen, ioloop +from zoonado.recipes.allocator import round_robin + + +log = logging.getLogger(__name__) + + +ANIMALS = ["cat", "dog", "mouse", "human"] + + +def arguments(parser): + parser.add_argument( + "znode", type=str, + help="Path of the base znode to use." + ) + parser.add_argument( + "--workers", "-w", type=int, default=5, + help="Number of worker coroutines to launch." + ) + parser.add_argument( + "--items", "-n", type=int, default=17, + help="Number of items to allocate amongst the workers." + ) + parser.add_argument( + "--alloc-func", "-a", default="round_robin", + choices=allocation_functions.keys(), + help="Which allocation function to use." + ) + + +@gen.coroutine +def run(client, args): + items = set([ + "%s::%s" % (i, random.choice(ANIMALS)) + for i in range(args.items) + ]) + allocation_function = allocation_functions[args.alloc_func] + + yield client.start() + + for i in range(args.workers): + ioloop.IOLoop.current().add_callback( + worker, i, client, args.znode, allocation_function, items + ) + + yield gen.sleep(10) + + +@gen.coroutine +def worker(number, client, znode_path, allocation_fn, items): + name = "worker_%s" % number + + allocator = client.recipes.Allocator(znode_path, name, allocation_fn) + + yield allocator.start() + + yield allocator.update(items) + + while True: + log.info("[WORKER %d] My set: %s", number, allocator.allocation) + yield gen.sleep(2) + + +def animal_buckets(members, items): + animal_assignment = {} + for member, animal in zip(itertools.cycle(members), ANIMALS): + animal_assignment[animal] = member + + allocation = collections.defaultdict(set) + for member, item in zip(itertools.cycle(members), items): + animal = item.split("::")[1] + allocation[animal_assignment[animal]].add(item) + + return allocation + + +allocation_functions = { + "round_robin": round_robin, + "buckets": animal_buckets, +} diff --git a/examples/barrier.py b/examples/barrier.py new file mode 100644 index 0000000..aebbc41 --- /dev/null +++ b/examples/barrier.py @@ -0,0 +1,52 @@ +import logging + +from tornado import gen, ioloop + + +log = logging.getLogger() + + +def arguments(parser): + parser.add_argument( + "--path", "-b", type=str, default="/example-barrier", + help="ZNode path to use for the barrier." + ) + parser.add_argument( + "--workers", "-w", type=int, default=5, + help="Number of worker coroutines." + ) + + +@gen.coroutine +def run(client, args): + yield client.start() + + barrier = client.recipes.Barrier(args.path) + + yield barrier.create() + + for i in range(args.workers): + yield gen.sleep(1) + ioloop.IOLoop.current().add_callback(worker, i, client, args.path) + + yield gen.sleep(2) + + yield barrier.lift() + + yield gen.sleep(3) + + yield client.close() + + +@gen.coroutine +def worker(number, client, barrier_path): + log.info("[WORKER #%d] Starting up", number) + + barrier = client.recipes.Barrier(barrier_path) + + log.info("[WORKER #%d] Waiting on barrier...", number) + yield barrier.wait() + + while True: + log.info("[WORKER #%d] Doing work!", number) + yield gen.sleep(.3) diff --git a/examples/child_watcher.py b/examples/child_watcher.py new file mode 100644 index 0000000..2d318bb --- /dev/null +++ b/examples/child_watcher.py @@ -0,0 +1,42 @@ +import logging +import random +from tornado import gen +from zoonado import exc + +log = logging.getLogger() + + +def arguments(parser): + parser.add_argument( + "--path", "-p", type=str, default="/examplewatcher", + help="ZNode path to use for the example." + ) + + +def watcher_callback(children): + children.sort() + log.info("There are %d items now: %s", len(children), children) + + +@gen.coroutine +def run(client, args): + yield client.start() + + try: + yield client.create(args.path) + except exc.NodeExists: + pass + + watcher = client.recipes.ChildrenWatcher() + + watcher.add_callback(args.path, watcher_callback) + + to_make = ["cat", "dog", "mouse", "human"] + random.shuffle(to_make) + + for item in to_make: + yield client.create(args.path + "/" + item, ephemeral=True) + yield gen.sleep(1) + + for item in to_make: + yield client.delete(args.path + "/" + item) diff --git a/examples/counter.py b/examples/counter.py new file mode 100644 index 0000000..9508b99 --- /dev/null +++ b/examples/counter.py @@ -0,0 +1,71 @@ +import logging +import random + +from tornado import gen +from zoonado import exc + + +log = logging.getLogger(__name__) + + +def arguments(parser): + parser.add_argument( + "--path", "-b", type=str, default="/example-counter", + help="ZNode path to use for the barrier." + ) + parser.add_argument( + "--workers", "-w", type=int, default=5, + help="Number of worker coroutines." + ) + + +@gen.coroutine +def run(client, args): + yield client.start() + + value_history = [] + + def callback(new_value): + value_history.append(new_value) + + watcher = client.recipes.DataWatcher() + watcher.add_callback(args.path, callback) + + try: + data = yield client.get_data(args.path) + log.info("Initial value is %s", data) + except exc.NoNode: + log.info("Initial value is blank") + + yield [ + worker(number, client, args) + for number in range(args.workers) + ] + + log.info("Value history: %s", " -> ".join(value_history)) + + yield client.close() + + +@gen.coroutine +def worker(number, client, args): + log.info("[WORKER #%d] Starting up", number) + + counter = client.recipes.Counter(args.path) + + yield counter.start() + + for i in range(2): + op = random.choice(["incr", "incr", "decr", "decr", "set"]) + if op == "incr": + log.info("[WORKER #%d] Incrementing count", number) + yield counter.incr() + elif op == "decr": + log.info("[WORKER #%d] Decrementing count", number) + yield counter.decr() + else: + new_value = random.choice([4, 8, 12, 16]) + log.info("[WORKER #%d] Setting count to %d", number, new_value) + yield counter.set_value(new_value) + + yield counter.stop() diff --git a/examples/data_watcher.py b/examples/data_watcher.py new file mode 100644 index 0000000..4846489 --- /dev/null +++ b/examples/data_watcher.py @@ -0,0 +1,38 @@ +import logging +import random +from tornado import gen +from zoonado import exc + +log = logging.getLogger() + + +def arguments(parser): + parser.add_argument( + "--path", "-p", type=str, default="/examplewatcher", + help="ZNode path to use for the example." + ) + + +def watcher_callback(new_data): + left, right = new_data.split(":") + log.info("Left: %s, Right: %s", left, right) + + +@gen.coroutine +def run(client, args): + yield client.start() + + try: + yield client.create(args.path) + except exc.NodeExists: + pass + + watcher = client.recipes.DataWatcher() + + watcher.add_callback(args.path, watcher_callback) + + choices = ["foo:bar", "bwee:bwoo", "derp:hork"] + + for i in range(5): + yield client.set_data(args.path, data=random.choice(choices)) + yield gen.sleep(1) diff --git a/examples/double_barrier.py b/examples/double_barrier.py new file mode 100644 index 0000000..339760a --- /dev/null +++ b/examples/double_barrier.py @@ -0,0 +1,60 @@ +import logging +import random + +from tornado import gen + + +log = logging.getLogger() + + +def arguments(parser): + parser.add_argument( + "--path", "-b", type=str, default="/example-barrier", + help="ZNode path to use for the double barrier." + ) + parser.add_argument( + "--workers", "-w", type=int, default=5, + help="Number of worker coroutines." + ) + parser.add_argument( + "--min-workers", "-m", type=int, default=3, + help="Minimum number of workers required to lift the barrier." + ) + + +@gen.coroutine +def run(client, args): + yield client.start() + + workers = [] + + for i in range(args.workers): + workers.append(worker(i, client, args)) + yield gen.sleep(1) + + yield workers + + yield client.close() + + +@gen.coroutine +def worker(number, client, args): + log.info("[WORKER #%d] Starting up", number) + + barrier = client.recipes.DoubleBarrier(args.path, args.min_workers) + + log.info("[WORKER #%d] Entering barrier...", number) + yield barrier.enter() + + workload = random.choice([3, 4, 5]) + + log.info("[WORKER #%d] My workload is %d", number, workload) + + for i in range(workload): + log.info("[WORKER #%d] Doing task %d", number, i + 1) + yield gen.sleep(1) + + log.info("[WORKER #%d] Workload complete, leaving barrier", number) + yield barrier.leave() + + log.info("[WORKER #%d] All done!", number) diff --git a/examples/election.py b/examples/election.py new file mode 100644 index 0000000..a486f5f --- /dev/null +++ b/examples/election.py @@ -0,0 +1,45 @@ +import logging +import random + +from tornado import gen + + +log = logging.getLogger() + + +def arguments(parser): + parser.add_argument( + "--workers", "-w", type=int, default=5, + help="Number of workers to launch." + ) + parser.add_argument( + "--znode-path", "-p", type=str, default="examplelock", + help="ZNode path to use for the election." + ) + + +@gen.coroutine +def run(client, args): + log.info("Launching %d workers.", args.workers) + yield client.start() + + order = list(range(args.workers)) + random.shuffle(order) + + yield [worker(i, client, args) for i in order] + + yield client.close() + + +@gen.coroutine +def worker(number, client, args): + election = client.recipes.LeaderElection(args.znode_path) + + yield election.join() + + if election.has_leadership: + log.info("[WORKER #%d] I am the leader!", number) + else: + log.info("[WORKER #%d] not the leader.", number) + + yield election.resign() diff --git a/examples/lease.py b/examples/lease.py new file mode 100644 index 0000000..4d389af --- /dev/null +++ b/examples/lease.py @@ -0,0 +1,47 @@ +import datetime +import logging +import random + +from tornado import gen, ioloop +from zoonado import exc + + +log = logging.getLogger() + + +def arguments(parser): + parser.add_argument( + "--path", "-p", type=str, default="/example-task-lease" + ) + parser.add_argument( + "--limit", "-l", type=int, default=1, + help="Max number of simultaneous leases." + ) + + +@gen.coroutine +def run(client, args): + yield client.start() + + try: + yield client.create(args.path) + except exc.NodeExists: + pass + + # simulate a cron job, same task fired at an interval + for i in range(8): + ioloop.IOLoop.current().add_callback(limited_task, i, client, args) + yield gen.sleep(1) + + +@gen.coroutine +def limited_task(number, client, args): + lease = client.recipes.Lease(args.path, args.limit) + + seconds = random.choice([1, 2, 3]) + + obtained = yield lease.obtain(duration=datetime.timedelta(seconds=seconds)) + if obtained: + log.info("[ITERATION #%d] Got lease for %d seconds", number, seconds) + else: + log.info("[ITERATION #%d] No lease available, can't work", number) diff --git a/examples/locking.py b/examples/locking.py new file mode 100644 index 0000000..a36d4a2 --- /dev/null +++ b/examples/locking.py @@ -0,0 +1,54 @@ +import logging +import random + +from tornado import gen + + +log = logging.getLogger() + + +def arguments(parser): + parser.add_argument( + "--workers", "-w", type=int, default=3, + help="Number of workers to launch." + ) + parser.add_argument( + "--lock-path", "-p", type=str, default="examplelock", + help="ZNode path to use for the lock." + ) + + +@gen.coroutine +def run(client, args): + log.info("Launching %d workers.", args.workers) + + yield client.start() + + order = list(range(args.workers)) + random.shuffle(order) + + yield [work(i, client, args) for i in order] + + yield client.close() + + +@gen.coroutine +def work(number, client, args): + lock = client.recipes.Lock(args.lock_path) + + num_iterations = 3 + + log.info("[WORKER #%d] Acquiring lock...", number) + with (yield lock.acquire()) as check: + log.info("[WORKER #%d] Got lock!", number) + + for i in range(num_iterations): + wait = random.choice([1, 2, 3]) + if not check(): + log.warn("[WORKER #%d] lost my lock!", number) + break + + log.info("[WORKER #%d] working %d secs", number, wait) + yield gen.sleep(wait) + + log.info("[WORKER #%d] Done!", number) diff --git a/examples/party.py b/examples/party.py new file mode 100644 index 0000000..f09134a --- /dev/null +++ b/examples/party.py @@ -0,0 +1,51 @@ +import logging +import random + +from tornado import gen + + +log = logging.getLogger() + + +def arguments(parser): + parser.add_argument( + "--workers", "-w", type=int, default=5, + help="Number of workers to launch." + ) + parser.add_argument( + "--znode-path", "-p", type=str, default="examplelock", + help="ZNode path to use for the election." + ) + + +@gen.coroutine +def run(client, args): + log.info("Launching %d workers.", args.workers) + yield client.start() + + yield [ + worker(i, client, args) + for i in range(args.workers) + ] + + yield client.close() + + +@gen.coroutine +def worker(number, client, args): + party = client.recipes.Party(args.znode_path, "worker_%d" % number) + + log.info("[WORKER #%d] Joining the party", number) + + yield party.join() + + for i in range(10): + log.info("[WORKER #%d] Members I see: %s", number, party.members) + yield gen.sleep(.5) + should_leave = random.choice([False, False, True]) + if should_leave: + log.info("[WORKER #%d] Leaving the party temporarily", number) + yield party.leave() + yield gen.sleep(1) + log.info("[WORKER #%d] Rejoining the party", number) + yield party.join() diff --git a/examples/run b/examples/run new file mode 100755 index 0000000..773a2ac --- /dev/null +++ b/examples/run @@ -0,0 +1,98 @@ +#!/usr/bin/env python +import argparse +import logging + +from tornado import ioloop +from zoonado import Zoonado + + +examples_list = ( + "runtime_config", + "locking", + "shared_locking", + "election", + "allocator", + "barrier", + "double_barrier", + "child_watcher", + "data_watcher", + "party", + "transactions", + "lease", + "counter", +) + + +def get_target_module(example): + try: + return __import__( + ".".join(["examples", example]), fromlist=["run", "arguments"] + ) + except (ImportError, AttributeError, ValueError) as e: + print("Error loading example '%s': %s" % (example, str(e))) + return None + + +example_xref = { + name: get_target_module(name) + for name in examples_list +} + + +parser = argparse.ArgumentParser() +parser.add_argument( + "--verbose", "-v", action="count", + help="Verbosity level. One for debug messages, two for protocol payloads." +) +parser.add_argument( + "--servers", default="localhost", + help="Comma-delimited list of zookeeper hosts." +) +parser.add_argument( + "--chroot", default=None, + help="Use a chroot path for the example." +) + +subparsers = parser.add_subparsers( + title="examples", description="available examples", dest="example", + help="Which available example to run." +) +for name, module in example_xref.items(): + if not module: + continue + subparser = subparsers.add_parser(name, help=module.__doc__) + module.arguments(subparser) + + +def main(): + args = parser.parse_args() + + logging.basicConfig(level=logging.INFO) + + if args.verbose >= 1: + logging.basicConfig(level=logging.DEBUG) + logging.getLogger("zoonado").setLevel(logging.DEBUG) + if args.verbose >= 2: + logging.getLogger("zoonado.connection.payload").setLevel(logging.DEBUG) + + example = example_xref[args.example] + + client = Zoonado(args.servers, chroot=args.chroot) + + loop = ioloop.IOLoop.instance() + + def wind_down(f): + try: + f.result() + finally: + loop.stop() + + loop.add_future(example.run(client, args), wind_down) + + try: + loop.start() + except KeyboardInterrupt: + loop.stop() + +if __name__ == "__main__": + main() diff --git a/examples/runtime_config.py b/examples/runtime_config.py new file mode 100644 index 0000000..bafb2f5 --- /dev/null +++ b/examples/runtime_config.py @@ -0,0 +1,93 @@ +import logging + +from tornado import ioloop, gen +from zoonado import exc + + +log = logging.getLogger() + + +def arguments(parser): + pass + + +@gen.coroutine +def run(client, args): + config_path = "/exampleconfig" + loop = ioloop.IOLoop.current() + + yield client.start() + + config = client.recipes.TreeCache(config_path) + + yield config.start() + + try: + yield client.create(config_path + "/running", data="yes") + except exc.NodeExists: + yield client.set_data(config_path + "/running", data="yes") + + for path in ["foo", "bar", "bazz", "bloo"]: + try: + yield client.create(config_path + "/" + path, data="1") + except exc.NodeExists: + yield client.set_data(config_path + "/" + path, data="1") + + loop.add_callback(foo, config) + loop.add_callback(bar, config) + loop.add_callback(bazz, config) + loop.add_callback(bloo, config) + + yield gen.sleep(1) + + yield client.set_data(config_path + "/foo", "3") + + yield gen.sleep(1) + + yield client.set_data(config_path + "/bar", "2") + + yield client.set_data(config_path + "/bazz", "5") + + yield gen.sleep(6) + + yield client.set_data(config_path + "/running", data="no") + + yield gen.sleep(2) + + yield client.close() + + +@gen.coroutine +def foo(config): + while config.running.value == "yes": + log.info("[FOO] doing work for %s seconds!", config.foo.value) + yield gen.sleep(int(config.foo.value)) + + log.info("[FOO] no longer working.") + + +@gen.coroutine +def bar(config): + while config.running.value == "yes": + log.info("[BAR] doing work for %s seconds!", config.bar.value) + yield gen.sleep(int(config.bar.value)) + + log.info("[BAR] no longer working.") + + +@gen.coroutine +def bazz(config): + while config.running.value == "yes": + log.info("[BAZZ] doing work for %s seconds!", config.bazz.value) + yield gen.sleep(int(config.bazz.value)) + + log.info("[BAZZ] no longer working.") + + +@gen.coroutine +def bloo(config): + while config.running.value == "yes": + log.info("[BLOO] doing work for %s seconds!", config.bloo.value) + yield gen.sleep(int(config.bloo.value)) + + log.info("[BLOO] no longer working.") diff --git a/examples/shared_locking.py b/examples/shared_locking.py new file mode 100644 index 0000000..7876a2c --- /dev/null +++ b/examples/shared_locking.py @@ -0,0 +1,58 @@ +import logging +import random + +from tornado import gen + +log = logging.getLogger() + + +def arguments(parser): + parser.add_argument( + "--reader-count", "-r", type=int, default=6, + help="Number of readers to launch.", + ) + parser.add_argument( + "--writer-count", "-w", type=int, default=2, + help="Number of writers to launch.", + ) + + +@gen.coroutine +def run(client, args): + workers = ([reader] * args.reader_count) + ([writer] * args.writer_count) + random.shuffle(workers) + + yield client.start() + + yield [ + worker_func(i, client) + for i, worker_func in enumerate(workers) + ] + + yield client.close() + + +@gen.coroutine +def reader(number, client): + lock = client.recipes.SharedLock("/examplelock") + + wait = random.choice([1, 2, 3]) + + log.info("[READER #%d] Acquiring lock...", number) + with (yield lock.acquire_read()): + log.info("[READER #%d] Got lock! Sleeping %d seconds", number, wait) + yield gen.sleep(wait) + log.info("[READER #%d] Done!", number) + + +@gen.coroutine +def writer(number, client): + lock = client.recipes.SharedLock("/examplelock") + + wait = random.choice([2, 3]) + + log.info("[WRITER #%d] Acquiring lock...", number) + with (yield lock.acquire_write()): + log.info("[WRITER #%d] Got lock! Sleeping %d seconds", number, wait) + yield gen.sleep(wait) + log.info("[WRITER #%d] Done!", number) diff --git a/examples/transactions.py b/examples/transactions.py new file mode 100644 index 0000000..41c394d --- /dev/null +++ b/examples/transactions.py @@ -0,0 +1,123 @@ +import logging +import random +import threading + +from tornado import gen, ioloop +from zoonado import Zoonado + + +log = logging.getLogger() + +monitor_ioloop = None + + +def arguments(parser): + pass + + +@gen.coroutine +def run(client, args): + yield client.start() + + yield client.create("/shared-znode", ephemeral=True) + + monitor_thread = threading.Thread(target=monitor_data, args=(args,)) + monitor_thread.start() + + threads = [ + threading.Thread(name="A", target=launch_loop, args=(args,)), + threading.Thread(name="B", target=launch_loop, args=(args,)), + threading.Thread(name="C", target=launch_loop, args=(args,)), + ] + + for thread in threads: + thread.start() + + for thread in threads: + thread.join() + + monitor_ioloop.stop() + + monitor_thread.join() + + yield client.close() + + +def monitor_data(args): + global monitor_ioloop + + name = threading.current_thread().name + log.info("Launching loop in thread %s", name) + + io_loop = ioloop.IOLoop() + + io_loop.make_current() + + monitor_ioloop = io_loop + + @gen.coroutine + def monitor(): + client = Zoonado(args.servers, chroot=args.chroot) + yield client.start() + + def data_callback(new_data): + log.info("Shared data set to '%s'", new_data) + + watcher = client.recipes.DataWatcher() + watcher.add_callback("/shared-znode", data_callback) + + yield gen.moment + + io_loop.add_callback(monitor) + + io_loop.start() + + +def launch_loop(args): + name = threading.current_thread().name + log.info("Launching loop in thread %s", name) + + io_loop = ioloop.IOLoop() + + io_loop.make_current() + + io_loop.add_callback(update_loop, name, args, io_loop) + + io_loop.start() + + +@gen.coroutine +def update_loop(name, args, io_loop): + log.info("[LOOP %s] starting up!", name) + + client = Zoonado(args.servers, chroot=args.chroot) + + try: + yield client.start() + + for i in range(5): + yield client.exists("/shared-znode") + expected_version = client.stat_cache["/shared-znode"].version + + yield gen.sleep(random.choice([.2, .4, .5])) + + log.info( + "[LOOP %s] I expect the shared znode to have version %s", + name, client.stat_cache["/shared-znode"].version + ) + + txn = client.begin_transaction() + + txn.create("/znode-" + name, ephemeral=True) + txn.check_version("/shared-znode", expected_version) + txn.set_data("/shared-znode", "altered by loop %s!" % name) + txn.delete("/znode-" + name) + + log.info("[LOOP %s] committing...", name) + result = yield txn.commit() + if not result: + log.info("[LOOP %s] rolled back!", name) + + yield client.close() + finally: + io_loop.stop() diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..187b8d7 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,4 @@ +[build_sphinx] +source_dir = docs +build_dir = .docbuild +all_files = 1 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..fbacc49 --- /dev/null +++ b/setup.py @@ -0,0 +1,61 @@ +from setuptools import setup, find_packages + +from zoonado import __version__ + + +setup( + name="zoonado", + version=__version__, + description="Async tornado client for Zookeeper.", + author="William Glass", + author_email="william.glass@gmail.com", + url="http://github.com/wglass/zoonado", + license="Apache", + keywords=["zookeeper", "tornado", "async", "distributed"], + packages=find_packages(exclude=["tests", "tests.*"]), + install_requires=[ + "tornado>=4.1", + "six" + ], + entry_points={ + "zoonado.recipes": [ + "data_watcher = zoonado.recipes.data_watcher:DataWatcher", + "children_watcher = zoonado.recipes.children_watcher:ChildrenWatcher", + "lock = zoonado.recipes.lock:Lock", + "shared_lock = zoonado.recipes.shared_lock:SharedLock", + "lease = zoonado.recipes.lease:Lease", + "barrier = zoonado.recipes.barrier:Barrier", + "double_barrier = zoonado.recipes.double_barrier:DoubleBarrier", + "election = zoonado.recipes.election:LeaderElection", + "party = zoonado.recipes.party:Party", + "counter = zoonado.recipes.counter:Counter", + "tree_cache = zoonado.recipes.tree_cache:TreeCache", + "allocator = zoonado.recipes.allocator:Allocator", + ], + }, + tests_require=[ + "nose", + "mock", + "coverage", + "flake8", + ], + classifiers=[ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Operating System :: MacOS", + "Operating System :: MacOS :: MacOS X", + "Operating System :: POSIX", + "Operating System :: POSIX :: Linux", + "Operating System :: Unix", + "Programming Language :: Python", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: Implementation", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Topic :: Software Development", + "Topic :: Software Development :: Libraries", + ], +) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/client_tests.py b/tests/client_tests.py new file mode 100644 index 0000000..356f40d --- /dev/null +++ b/tests/client_tests.py @@ -0,0 +1,21 @@ +import unittest + +from zoonado.protocol.acl import ACL + +from zoonado import client + + +class ClientTests(unittest.TestCase): + + def test_default_acl_is_unrestricted(self): + c = client.Zoonado("host,host,host") + + self.assertEqual(len(c.default_acl), 1) + + self.assertEqual( + c.default_acl[0], + ACL.make( + scheme="world", id="anyone", + read=True, write=True, create=True, delete=True, admin=True + ) + ) diff --git a/tests/style_tests.py b/tests/style_tests.py new file mode 100644 index 0000000..3d1ac46 --- /dev/null +++ b/tests/style_tests.py @@ -0,0 +1,48 @@ +import os + +import flake8.main +import six + + +MAX_COMPLEXITY = 11 + + +def test_style(): + for path in ("zoonado", "tests", "examples"): + python_files = list(get_python_files(path)) + yield create_style_assert(path, python_files) + + +def get_python_files(path): + path = os.path.join(os.path.dirname(__file__), "../", path) + for root, dirs, files in os.walk(path): + for filename in files: + if not filename.endswith(".py"): + continue + yield os.path.join(root, filename) + + +def create_style_assert(path, python_files): + + def test_function(): + assert_conforms_to_style(python_files) + + test_name = "test_style__%s" % path + test_function.__name__ = test_name + test_function.description = test_name + + return test_function + + +def assert_conforms_to_style(python_files): + checker = flake8.main.get_style_guide( + paths=python_files, max_complexity=MAX_COMPLEXITY + ) + checker.options.jobs = 1 + checker.options.verbose = True + result = checker.check_files() + + assert not result.messages, "\n".join([ + "%s: %s" % (code, message) + for code, message in six.iteritems(result.messages) + ]) diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..e14fe84 --- /dev/null +++ b/tox.ini @@ -0,0 +1,13 @@ +[tox] +envlist = py27,py34,pypy +skipsdist = True + +[testenv] +usedevelop = True +deps = + tornado + nose + coverage + mock + flake8 +commands = nosetests {toxinidir}/tests --with-coverage --cover-package=zoonado diff --git a/zoonado/__init__.py b/zoonado/__init__.py new file mode 100644 index 0000000..07b0024 --- /dev/null +++ b/zoonado/__init__.py @@ -0,0 +1,8 @@ +from .client import Zoonado # noqa +from .protocol import WatchEvent # noqa +from .protocol.acl import ACL # noqa +from .retry import RetryPolicy # noqa + +version_info = (0, 8, 0) + +__version__ = ".".join((str(point) for point in version_info)) diff --git a/zoonado/client.py b/zoonado/client.py new file mode 100644 index 0000000..51cd92e --- /dev/null +++ b/zoonado/client.py @@ -0,0 +1,230 @@ +import logging + +from tornado import gen, concurrent + +from zoonado import protocol, exc + +from .recipes.proxy import RecipeProxy +from .session import Session +from .transaction import Transaction +from .features import Features + + +log = logging.getLogger(__name__) + + +class Zoonado(object): + + def __init__( + self, + servers, + chroot=None, + session_timeout=10, + default_acl=None, + retry_policy=None, + allow_read_only=False, + ): + self.chroot = None + if chroot: + self.chroot = self.normalize_path(chroot) + log.info("Using chroot '%s'", self.chroot) + + self.session = Session( + servers, session_timeout, retry_policy, allow_read_only + ) + + self.default_acl = default_acl or [protocol.UNRESTRICTED_ACCESS] + + self.stat_cache = {} + + self.recipes = RecipeProxy(self) + + def normalize_path(self, path): + if self.chroot: + path = "/".join([self.chroot, path]) + + normalized = "/".join([ + name for name in path.split("/") + if name + ]) + + return "/" + normalized + + def denormalize(self, path): + if self.chroot: + path = path[len(self.chroot):] + + return path + + @gen.coroutine + def start(self): + yield self.session.start() + + if self.chroot: + yield self.ensure_path("/") + + @property + def features(self): + if self.session.conn: + return Features(self.session.conn.version_info) + else: + return Features((0, 0, 0)) + + @gen.coroutine + def send(self, request): + response = yield self.session.send(request) + + if getattr(request, "path", None) and getattr(response, "stat", None): + self.stat_cache[self.denormalize(request.path)] = response.stat + + raise gen.Return(response) + + @gen.coroutine + def close(self): + yield self.session.close() + + def wait_for_event(self, event_type, path): + path = self.normalize_path(path) + + f = concurrent.Future() + + def set_future(_): + if not f.done(): + f.set_result(None) + self.session.remove_watch_callback(event_type, path, set_future) + + self.session.add_watch_callback(event_type, path, set_future) + + return f + + @gen.coroutine + def exists(self, path, watch=False): + path = self.normalize_path(path) + + try: + yield self.send(protocol.ExistsRequest(path=path, watch=watch)) + except exc.NoNode: + raise gen.Return(False) + + raise gen.Return(True) + + @gen.coroutine + def create( + self, path, data=None, acl=None, + ephemeral=False, sequential=False, container=False, + ): + if container and not self.features.containers: + raise ValueError("Cannot create container, feature unavailable.") + + path = self.normalize_path(path) + acl = acl or self.default_acl + + if self.features.create_with_stat: + request_class = protocol.Create2Request + else: + request_class = protocol.CreateRequest + + request = request_class(path=path, data=data, acl=acl) + request.set_flags(ephemeral, sequential, container) + + response = yield self.send(request) + + raise gen.Return(self.denormalize(response.path)) + + @gen.coroutine + def ensure_path(self, path, acl=None): + path = self.normalize_path(path) + + acl = acl or self.default_acl + + paths_to_make = [] + for segment in path[1:].split("/"): + if not paths_to_make: + paths_to_make.append("/" + segment) + continue + + paths_to_make.append("/".join([paths_to_make[-1], segment])) + + while paths_to_make: + path = paths_to_make[0] + + if self.features.create_with_stat: + request = protocol.Create2Request(path=path, acl=acl) + else: + request = protocol.CreateRequest(path=path, acl=acl) + request.set_flags( + ephemeral=False, sequential=False, + container=self.features.containers + ) + + try: + yield self.send(request) + except exc.NodeExists: + pass + + paths_to_make.pop(0) + + @gen.coroutine + def delete(self, path, force=False): + path = self.normalize_path(path) + + if not force and path in self.stat_cache: + version = self.stat_cache[path].version + else: + version = -1 + + yield self.send(protocol.DeleteRequest(path=path, version=version)) + + @gen.coroutine + def get_data(self, path, watch=False): + path = self.normalize_path(path) + + response = yield self.send( + protocol.GetDataRequest(path=path, watch=watch) + ) + raise gen.Return(response.data) + + @gen.coroutine + def set_data(self, path, data, force=False): + path = self.normalize_path(path) + + if not force and path in self.stat_cache: + version = self.stat_cache[path].version + else: + version = -1 + + yield self.send( + protocol.SetDataRequest(path=path, data=data, version=version) + ) + + @gen.coroutine + def get_children(self, path, watch=False): + path = self.normalize_path(path) + + response = yield self.send( + protocol.GetChildren2Request(path=path, watch=watch) + ) + raise gen.Return(response.children) + + @gen.coroutine + def get_acl(self, path): + path = self.normalize_path(path) + + response = yield self.send(protocol.GetACLRequest(path=path)) + raise gen.Return(response.acl) + + @gen.coroutine + def set_acl(self, path, acl, force=False): + path = self.normalize_path(path) + + if not force and path in self.stat_cache: + version = self.stat_cache[path].version + else: + version = -1 + + yield self.send( + protocol.SetACLRequest(path=path, acl=acl, version=version) + ) + + def begin_transaction(self): + return Transaction(self) diff --git a/zoonado/connection.py b/zoonado/connection.py new file mode 100644 index 0000000..701a429 --- /dev/null +++ b/zoonado/connection.py @@ -0,0 +1,236 @@ +import collections +import logging +import re +import struct +import sys + +from tornado import ioloop, iostream, gen, concurrent, tcpclient + +from zoonado import protocol, iterables, exc + + +version_regex = re.compile(r'Zookeeper version: (\d)\.(\d)\.(\d)-.*') + +# all requests and responses are prefixed with a 32-bit int denoting size +size_struct = struct.Struct("!i") +# replies are prefixed with an xid, zxid and error code +reply_header_struct = struct.Struct("!iqi") + +log = logging.getLogger(__name__) +payload_log = logging.getLogger(__name__ + ".payload") +if payload_log.level == logging.NOTSET: + payload_log.setLevel(logging.INFO) + + +class Connection(object): + + def __init__(self, host, port, watch_handler): + self.host = host + self.port = int(port) + + self.stream = None + self.closing = False + + self.version_info = None + self.start_read_only = None + + self.watch_handler = watch_handler + + self.opcode_xref = {} + + self.pending = {} + self.pending_specials = collections.defaultdict(list) + + self.watches = collections.defaultdict(list) + + @gen.coroutine + def connect(self): + client = tcpclient.TCPClient() + + log.debug("Initial connection to server %s:%d", self.host, self.port) + stream = yield client.connect(self.host, self.port) + + log.debug("Sending 'srvr' command to %s:%d", self.host, self.port) + yield stream.write("srvr") + answer = yield stream.read_until_close() + + version_line = answer.split("\n")[0] + self.version_info = tuple( + map(int, version_regex.match(version_line).groups()) + ) + self.start_read_only = bool("READ_ONLY" in answer) + + log.debug("Version info: %s", self.version_info) + log.debug("Read-only mode: %s", self.start_read_only) + + log.debug("Actual connection to server %s:%d", self.host, self.port) + self.stream = yield client.connect(self.host, self.port) + + @gen.coroutine + def send_connect(self, request): + # meant to be used before the read_loop starts + payload_log.debug("[SEND] (initial) %s", request) + + payload = request.serialize() + payload = size_struct.pack(len(payload)) + payload + + yield self.stream.write(payload) + + try: + _, zxid, response = yield self.read_response(initial_connect=True) + except Exception: + log.exception("Error reading connect response.") + return + + payload_log.debug("[RECV] (initial) %s", response) + + raise gen.Return((zxid, response)) + + def start_read_loop(self): + ioloop.IOLoop.current().add_callback(self.read_loop) + + def send(self, request, xid=None): + f = concurrent.Future() + + if self.closing: + f.set_exception(exc.ConnectError(self.host, self.port)) + return f + + if request.special_xid: + xid = request.special_xid + + payload_log.debug("[SEND] (xid: %s) %s", xid, request) + + payload = request.serialize(xid) + payload = size_struct.pack(len(payload)) + payload + + self.opcode_xref[xid] = request.opcode + + if xid in protocol.SPECIAL_XIDS: + self.pending_specials[xid].append(f) + else: + self.pending[xid] = f + + def handle_write(write_future): + try: + write_future.result() + except Exception: + self.abort() + + try: + self.stream.write(payload).add_done_callback(handle_write) + except Exception: + self.abort() + + return f + + @gen.coroutine + def read_loop(self): + """ + Infinite loop that reads messages off of the socket while not closed. + + When a message is received its corresponding pending Future is set + to have the message as its result. + + This is never used directly and is fired as a separate callback on the + I/O loop via the `connect()` method. + """ + while not self.closing: + try: + xid, zxid, response = yield self.read_response() + except iostream.StreamClosedError: + return + except Exception: + log.exception("Error reading response.") + self.abort() + return + + payload_log.debug("[RECV] (xid: %s) %s", xid, response) + + if xid == protocol.WATCH_XID: + self.watch_handler(response) + continue + elif xid in protocol.SPECIAL_XIDS: + f = self.pending_specials[xid].pop() + else: + f = self.pending.pop(xid) + + if isinstance(response, Exception): + f.set_exception(response) + else: + f.set_result((xid, response)) + + @gen.coroutine + def read_response(self, initial_connect=False): + raw_size = yield self.stream.read_bytes(size_struct.size) + size = size_struct.unpack(raw_size)[0] + + # connect and close op replies don't contain a reply header + if initial_connect or self.pending_specials[protocol.CLOSE_XID]: + raw_payload = yield self.stream.read_bytes(size) + response = protocol.ConnectResponse.deserialize(raw_payload) + raise gen.Return((None, None, response)) + + raw_header = yield self.stream.read_bytes(reply_header_struct.size) + xid, zxid, error_code = reply_header_struct.unpack_from(raw_header) + + if error_code: + raise gen.Return((xid, zxid, exc.get_response_error(error_code))) + + size -= reply_header_struct.size + + raw_payload = yield self.stream.read_bytes(size) + + if xid == protocol.WATCH_XID: + response = protocol.WatchEvent.deserialize(raw_payload) + else: + opcode = self.opcode_xref.pop(xid) + response = protocol.response_xref[opcode].deserialize(raw_payload) + + raise gen.Return((xid, zxid, response)) + + def abort(self, exception=exc.ConnectError): + """ + Aborts a connection and puts all pending futures into an error state. + + If ``sys.exc_info()`` is set (i.e. this is being called in an exception + handler) then pending futures will have that exc info set. Otherwise + the given ``exception`` parameter is used (defaults to + ``ConnectError``). + """ + log.warn("Aborting connection to %s:%s", self.host, self.port) + + def abort_pending(f): + exc_info = sys.exc_info() + if any(exc_info): + f.set_exc_info(exc_info) + else: + f.set_exception(exception(self.host, self.port)) + + for pending in self.drain_all_pending(): + abort_pending(pending) + + def drain_all_pending(self): + for special_xid in protocol.SPECIAL_XIDS: + for _, f in iterables.drain(self.pending_specials[special_xid]): + yield f + for _, f in iterables.drain(self.pending): + yield f + + @gen.coroutine + def close(self, timeout): + if self.closing: + return + + self.closing = True + + pending_with_timeouts = [] + for pending in self.drain_all_pending(): + pending_with_timeouts.append(gen.with_timeout(timeout, pending)) + + try: + yield list(pending_with_timeouts) + except gen.TimeoutError: + yield self.abort(exception=exc.TimeoutError) + finally: + self.stream.close() diff --git a/zoonado/exc.py b/zoonado/exc.py new file mode 100644 index 0000000..da1eddc --- /dev/null +++ b/zoonado/exc.py @@ -0,0 +1,180 @@ +class ZKError(Exception): + pass + + +class ConnectError(ZKError): + def __init__(self, host, port, server_id=None): + self.host = host + self.port = port + self.server_id = server_id + + def __str__(self): + return "Error connecting to %s:%s" % (self.host, self.port) + + +class NoServersError(ZKError): + pass + + +class SessionLost(ZKError): + pass + + +class InvalidClientState(ZKError): + pass + + +class TimeoutError(ZKError): + pass + + +class FailedRetry(ZKError): + pass + + +response_error_xref = {} + + +class ResponseErrorMeta(type): + + def __new__(cls, name, bases, attrs): + new_class = super(ResponseErrorMeta, cls).__new__( + cls, name, bases, attrs + ) + + response_error_xref[new_class.error_code] = new_class + + return new_class + + +class ResponseError(ZKError): + __metaclass__ = ResponseErrorMeta + + error_code = None + + def __str__(self): + return self.__class__.__name__ + + +class UnknownError(ResponseError): + + def __init__(self, error_code): + self.error_code = error_code + + def __str__(self): + return "Unknown error code: %s" % self.error_code + + +def get_response_error(error_code): + if error_code not in response_error_xref: + return UnknownError(error_code) + + return response_error_xref[error_code]() + + +class RolledBack(ResponseError): + error_code = 0 + + +class SystemError(ResponseError): + error_code = -1 + + +class RuntimeInconsistency(ResponseError): + error_code = -2 + + +class DataInconsistency(ResponseError): + error_code = -3 + + +class ConnectionLoss(ResponseError): + error_code = -4 + + +class MarshallingError(ResponseError): + error_code = -5 + + +class Unimplemented(ResponseError): + error_code = -6 + + +class OperationTimeout(ResponseError): + error_code = -7 + + +class BadArguments(ResponseError): + error_code = -8 + + +class UnknownSession(ResponseError): + error_code = -12 + + +class NewConfigNoQuorum(ResponseError): + error_code = -13 + + +class ReconfigInProcess(ResponseError): + error_code = -14 + + +class APIError(ResponseError): + error_code = -100 + + +class NoNode(ResponseError): + error_code = -101 + + +class NoAuth(ResponseError): + error_code = -102 + + +class BadVersion(ResponseError): + error_code = -103 + + +class NoChildrenForEphemerals(ResponseError): + error_code = -108 + + +class NodeExists(ResponseError): + error_code = -110 + + +class NotEmpty(ResponseError): + error_code = -111 + + +class SessionExpired(ResponseError): + error_code = -112 + + +class InvalidCallback(ResponseError): + error_code = -113 + + +class InvalidACL(ResponseError): + error_code = -114 + + +class AuthFailed(ResponseError): + error_code = -115 + + +class SessionMoved(ResponseError): + error_code = -118 + + +class NotReadOnly(ResponseError): + error_code = -119 + + +class EphemeralOnLocalSession(ResponseError): + error_code = -120 + + +class NoWatcher(ResponseError): + error_code = -121 diff --git a/zoonado/features.py b/zoonado/features.py new file mode 100644 index 0000000..134d9f2 --- /dev/null +++ b/zoonado/features.py @@ -0,0 +1,12 @@ +ALL_FEATURES = { + "create_with_stat": (3, 5, 0), + "containers": (3, 5, 1), + "reconfigure": (3, 5, 0), +} + + +class Features(object): + + def __init__(self, version_info): + for feature_name, version_introduced in ALL_FEATURES.items(): + setattr(self, feature_name, version_info >= version_introduced) diff --git a/zoonado/iterables.py b/zoonado/iterables.py new file mode 100644 index 0000000..547de6d --- /dev/null +++ b/zoonado/iterables.py @@ -0,0 +1,26 @@ +def drain(iterable): + """ + Helper method that empties an iterable as it is iterated over. + + Works for: + + * ``dict`` + * ``collections.deque`` + * ``list`` + * ``set`` + """ + if getattr(iterable, "popleft", False): + def next_item(coll): + return coll.popleft() + elif getattr(iterable, "popitem", False): + def next_item(coll): + return coll.popitem() + else: + def next_item(coll): + return coll.pop() + + while True: + try: + yield next_item(iterable) + except (IndexError, KeyError): + raise StopIteration diff --git a/zoonado/protocol/__init__.py b/zoonado/protocol/__init__.py new file mode 100644 index 0000000..f4d5250 --- /dev/null +++ b/zoonado/protocol/__init__.py @@ -0,0 +1,89 @@ +from .acl import ( # noqa + GetACLRequest, + GetACLResponse, + SetACLRequest, + SetACLResponse, + WORLD_READABLE, + AUTHED_UNRESTRICTED, + UNRESTRICTED_ACCESS, +) +from .auth import ( # noqa + AuthRequest, + AuthResponse, + AUTH_XID, +) +from .check import ( # noqa + CheckVersionRequest, + CheckVersionResponse, +) +from .children import ( # noqa + GetChildrenRequest, + GetChildrenResponse, + GetChildren2Request, + GetChildren2Response, +) +from .close import ( # noqa + CloseRequest, + CloseResponse, + CLOSE_XID, +) +from .connect import ( # noqa + ConnectRequest, + ConnectResponse, +) +from .create import ( # noqa + CreateRequest, + CreateResponse, + Create2Request, + Create2Response, +) +from .data import ( # noqa + GetDataRequest, + GetDataResponse, + SetDataRequest, + SetDataResponse, +) +from .delete import ( # noqa + DeleteRequest, + DeleteResponse, +) +from .exists import ( # noqa + ExistsRequest, + ExistsResponse, +) +from .ping import ( # noqa + PingRequest, + PingResponse, + PING_XID, +) +from .reconfig import ( # noqa + ReconfigRequest, + ReconfigResponse, +) +from .sasl import ( # noqa + SASLRequest, + SASLResponse, +) +from .sync import ( # noqa + SyncRequest, + SyncResponse, +) +from .watches import ( # noqa + WatchEvent, + SetWatchesRequest, + SetWatchesResponse, + CheckWatchesRequest, + CheckWatchesResponse, + RemoveWatchesRequest, + RemoveWatchesResponse, + WATCH_XID, +) +from .response import ( # noqa + response_xref +) +from .transaction import ( # noqa + TransactionRequest, + TransactionResponse, +) + +SPECIAL_XIDS = (AUTH_XID, PING_XID, CLOSE_XID) diff --git a/zoonado/protocol/acl.py b/zoonado/protocol/acl.py new file mode 100644 index 0000000..f3a751a --- /dev/null +++ b/zoonado/protocol/acl.py @@ -0,0 +1,113 @@ +from .request import Request +from .response import Response +from .part import Part +from .stat import Stat +from .primitives import UString, Int, Vector + + +class ID(Part): + """ + """ + parts = ( + ("scheme", UString), + ("id", UString), + ) + + +class ACL(Part): + """ + """ + READ_PERM = 1 << 0 + WRITE_PERM = 1 << 1 + CREATE_PERM = 1 << 2 + DELETE_PERM = 1 << 3 + ADMIN_PERM = 1 << 4 + + parts = ( + ("perms", Int), + ("id", ID), + ) + + @classmethod + def make( + cls, scheme, id, + read=False, write=False, create=False, delete=False, admin=False + ): + instance = cls(id=ID(scheme=scheme, id=id)) + instance.set_perms(read, write, create, delete, admin) + + return instance + + def set_perms(self, read, write, create, delete, admin): + perms = 0 + if read: + perms |= self.READ_PERM + if write: + perms |= self.WRITE_PERM + if create: + perms |= self.CREATE_PERM + if delete: + perms |= self.DELETE_PERM + if admin: + perms |= self.ADMIN_PERM + + self.perms = perms + + +WORLD_READABLE = ACL.make( + scheme="world", id="anyone", + read=True, write=False, create=False, delete=False, admin=False +) + +AUTHED_UNRESTRICTED = ACL.make( + scheme="auth", id="", + read=True, write=True, create=True, delete=True, admin=True +) + +UNRESTRICTED_ACCESS = ACL.make( + scheme="world", id="anyone", + read=True, write=True, create=True, delete=True, admin=True +) + + +class GetACLRequest(Request): + """ + """ + opcode = 6 + + parts = ( + ("path", UString), + ) + + +class GetACLResponse(Response): + """ + """ + opcode = 6 + + parts = ( + ("acl", Vector.of(ACL)), + ("stat", Stat), + ) + + +class SetACLRequest(Request): + """ + """ + opcode = 7 + + parts = ( + ("path", UString), + ("acl", Vector.of(ACL)), + ("version", Int), + ) + + +class SetACLResponse(Response): + """ + """ + opcode = 7 + + parts = ( + ("stat", Stat), + ) diff --git a/zoonado/protocol/auth.py b/zoonado/protocol/auth.py new file mode 100644 index 0000000..54675af --- /dev/null +++ b/zoonado/protocol/auth.py @@ -0,0 +1,27 @@ +from .request import Request +from .response import Response +from .primitives import Int, Buffer, UString + + +AUTH_XID = -4 + + +class AuthRequest(Request): + """ + """ + opcode = 100 + special_xid = AUTH_XID + + parts = ( + ("type", Int), + ("scheme", UString), + ("auth", Buffer), + ) + + +class AuthResponse(Response): + """ + """ + opcode = 100 + + parts = () diff --git a/zoonado/protocol/check.py b/zoonado/protocol/check.py new file mode 100644 index 0000000..20f57a4 --- /dev/null +++ b/zoonado/protocol/check.py @@ -0,0 +1,22 @@ +from .request import Request +from .response import Response +from .primitives import Int, UString + + +class CheckVersionRequest(Request): + """ + """ + opcode = 13 + + parts = ( + ("path", UString), + ("version", Int), + ) + + +class CheckVersionResponse(Response): + """ + """ + opcode = 13 + + parts = () diff --git a/zoonado/protocol/children.py b/zoonado/protocol/children.py new file mode 100644 index 0000000..9e82fef --- /dev/null +++ b/zoonado/protocol/children.py @@ -0,0 +1,47 @@ +from .request import Request +from .response import Response +from .stat import Stat +from .primitives import Bool, UString, Vector + + +class GetChildrenRequest(Request): + """ + """ + opcode = 8 + + parts = ( + ("path", UString), + ("watch", Bool), + ) + + +class GetChildrenResponse(Response): + """ + """ + opcode = 8 + + parts = ( + ("children", Vector.of(UString)), + ) + + +class GetChildren2Request(Request): + """ + """ + opcode = 12 + + parts = ( + ("path", UString), + ("watch", Bool), + ) + + +class GetChildren2Response(Response): + """ + """ + opcode = 12 + + parts = ( + ("children", Vector.of(UString)), + ("stat", Stat), + ) diff --git a/zoonado/protocol/close.py b/zoonado/protocol/close.py new file mode 100644 index 0000000..e410ec6 --- /dev/null +++ b/zoonado/protocol/close.py @@ -0,0 +1,22 @@ +from .request import Request +from .response import Response + + +CLOSE_XID = None + + +class CloseRequest(Request): + """ + """ + opcode = -11 + special_xid = CLOSE_XID + + parts = () + + +class CloseResponse(Response): + """ + """ + opcode = -11 + + parts = () diff --git a/zoonado/protocol/connect.py b/zoonado/protocol/connect.py new file mode 100644 index 0000000..1e0d91e --- /dev/null +++ b/zoonado/protocol/connect.py @@ -0,0 +1,27 @@ +from .request import Request +from .response import Response +from .primitives import Int, Long, Buffer, Bool + + +class ConnectRequest(Request): + """ + """ + parts = ( + ("protocol_version", Int), + ("last_seen_zxid", Long), + ("timeout", Int), + ("session_id", Long), + ("password", Buffer), + ("read_only", Bool), + ) + + +class ConnectResponse(Response): + """ + """ + parts = ( + ("protocol_version", Int), + ("timeout", Int), + ("session_id", Long), + ("password", Buffer), + ) diff --git a/zoonado/protocol/create.py b/zoonado/protocol/create.py new file mode 100644 index 0000000..38cc010 --- /dev/null +++ b/zoonado/protocol/create.py @@ -0,0 +1,62 @@ +from .request import Request +from .response import Response +from .stat import Stat +from .acl import ACL +from .primitives import UString, Int, Buffer, Vector + + +class CreateRequest(Request): + """ + """ + opcode = 1 + + writes = True + + EPHEMERAL_FLAG = 1 << 0 + SEQUENTIAL_FLAG = 1 << 1 + CONTAINER_FLAG = 1 << 2 + + parts = ( + ("path", UString), + ("data", Buffer), + ("acl", Vector.of(ACL)), + ("flags", Int), + ) + + def set_flags(self, ephemeral=False, sequential=False, container=False): + flags = 0 + if ephemeral: + flags |= self.EPHEMERAL_FLAG + if sequential: + flags |= self.SEQUENTIAL_FLAG + if container: + flags |= self.CONTAINER_FLAG + + self.flags = flags + + +class CreateResponse(Response): + """ + """ + opcode = 1 + + parts = ( + ("path", UString), + ) + + +class Create2Request(CreateRequest): + """ + """ + opcode = 15 + + +class Create2Response(Response): + """ + """ + opcode = 15 + + parts = ( + ("path", UString), + ("stat", Stat), + ) diff --git a/zoonado/protocol/data.py b/zoonado/protocol/data.py new file mode 100644 index 0000000..b239cd7 --- /dev/null +++ b/zoonado/protocol/data.py @@ -0,0 +1,50 @@ +from .request import Request +from .response import Response +from .stat import Stat +from .primitives import Bool, UString, Buffer, Int + + +class GetDataRequest(Request): + """ + """ + opcode = 4 + + parts = ( + ("path", UString), + ("watch", Bool), + ) + + +class GetDataResponse(Response): + """ + """ + opcode = 4 + + parts = ( + ("data", Buffer), + ("stat", Stat) + ) + + +class SetDataRequest(Request): + """ + """ + opcode = 5 + + writes = True + + parts = ( + ("path", UString), + ("data", Buffer), + ("version", Int), + ) + + +class SetDataResponse(Response): + """ + """ + opcode = 5 + + parts = ( + ("stat", Stat), + ) diff --git a/zoonado/protocol/delete.py b/zoonado/protocol/delete.py new file mode 100644 index 0000000..b14fc58 --- /dev/null +++ b/zoonado/protocol/delete.py @@ -0,0 +1,24 @@ +from .request import Request +from .response import Response +from .primitives import UString, Int + + +class DeleteRequest(Request): + """ + """ + opcode = 2 + + writes = True + + parts = ( + ("path", UString), + ("version", Int), + ) + + +class DeleteResponse(Response): + """ + """ + opcode = 2 + + parts = () diff --git a/zoonado/protocol/exists.py b/zoonado/protocol/exists.py new file mode 100644 index 0000000..b4602ca --- /dev/null +++ b/zoonado/protocol/exists.py @@ -0,0 +1,25 @@ +from .request import Request +from .response import Response +from .stat import Stat +from .primitives import UString, Bool + + +class ExistsRequest(Request): + """ + """ + opcode = 3 + + parts = ( + ("path", UString), + ("watch", Bool), + ) + + +class ExistsResponse(Response): + """ + """ + opcode = 3 + + parts = ( + ("stat", Stat), + ) diff --git a/zoonado/protocol/part.py b/zoonado/protocol/part.py new file mode 100644 index 0000000..10d5ff3 --- /dev/null +++ b/zoonado/protocol/part.py @@ -0,0 +1,100 @@ +from .primitives import Primitive + + +class Part(object): + """ + Composable building block used to define Zookeeper protocol parts. + + Behaves much like the `Primitive` class but has named "sub parts" + stored in a ``parts`` class attribute, that can hold any `Part` or + `Primitive` subclass. + """ + parts = () + + def __init__(self, **kwargs): + part_names = [item[0] for item in self.parts] + + for name, value in kwargs.items(): + if name not in part_names: + raise ValueError("Unknown part name: '%s'" % name) + + setattr(self, name, value) + + def render(self, parts=None): + """ + Returns a two-element tuple with the ``struct`` format and values. + + Iterates over the applicable sub-parts and calls `render()` on them, + accumulating the format string and values. + + Optionally takes a subset of parts to render, default behavior is to + render all sub-parts belonging to the class. + """ + if not parts: + parts = self.parts + + format = [] + data = [] + + for name, part_class in parts: + if issubclass(part_class, Primitive): + part = part_class(getattr(self, name, None)) + else: + part = getattr(self, name, None) + + part_format, part_data = part.render() + + format.extend(part_format) + data.extend(part_data) + + return "".join(format), data + + @classmethod + def parse(cls, buff, offset): + """ + Given a buffer and offset, returns the parsed value and new offset. + + Calls `parse()` on the given buffer for each sub-part in order and + creates a new instance with the results. + """ + values = {} + + for name, part in cls.parts: + value, new_offset = part.parse(buff, offset) + + values[name] = value + offset = new_offset + + return cls(**values), offset + + def __eq__(self, other): + """ + `Part` instances are equal if all of their sub-parts are also equal. + """ + try: + return all([ + getattr(self, part_name) == getattr(other, part_name) + for part_name, part_class in self.parts + ]) + except AttributeError: + return False + + def __str__(self): + + def subpart_string(part_info): + part_name, part_class = part_info + + if not part_class.__name__.startswith("VectorOf"): + return "%s=%s" % (part_name, getattr(self, part_name, None)) + + return "%s=[%s]" % ( + part_name, + ", ".join([ + str(item) for item in getattr(self, part_name, []) + ]) + ) + + return "%s(%s)" % ( + self.__class__.__name__, + ", ".join([subpart_string(part) for part in self.parts]) + ) diff --git a/zoonado/protocol/ping.py b/zoonado/protocol/ping.py new file mode 100644 index 0000000..0faf683 --- /dev/null +++ b/zoonado/protocol/ping.py @@ -0,0 +1,22 @@ +from .request import Request +from .response import Response + + +PING_XID = -2 + + +class PingRequest(Request): + """ + """ + opcode = 11 + special_xid = PING_XID + + parts = () + + +class PingResponse(Response): + """ + """ + opcode = 11 + + parts = () diff --git a/zoonado/protocol/primitives.py b/zoonado/protocol/primitives.py new file mode 100644 index 0000000..7852ca0 --- /dev/null +++ b/zoonado/protocol/primitives.py @@ -0,0 +1,255 @@ +import struct + + +class Primitive(object): + """ + The most basic structure of the protocol. Subclassed, never used directly. + + Used as a building block for the various actually-used primitives outlined + in the Zookeeper jute file: + + https://github.com/apache/zookeeper/blob/trunk/src/zookeeper.jute + """ + format = None + + def __init__(self, value): + self.value = value + + def render(self): + """ + Returns a two-element tuple with the ``struct`` format and list value. + + The value is wrapped in a list, as there are some primitives that deal + with multiple values. Any caller of `render()` should expect a list. + """ + return self.format, [self.value] + + @classmethod + def parse(cls, buff, offset): + """ + Given a buffer and offset, returns the parsed value and new offset. + + Uses the ``format`` class attribute to unpack the data from the buffer + and determine the used up number of bytes. + """ + primitive_struct = struct.Struct("!" + cls.format) + + value = primitive_struct.unpack_from(buff, offset)[0] + offset += primitive_struct.size + + return value, offset + + def __eq__(self, other): + """ + Basic equality method that tests equality of the ``value`` attributes. + """ + return self.value == other.value + + def __str__(self): + return "%s(%s)" % (self.__class__.__name__, self.value) + + +class VariablePrimitive(Primitive): + """ + Base primitive for variable-length scalar primitives (strings and bytes). + """ + size_primitive = None + + def render_value(self, value): + raise NotImplementedError + + @classmethod + def parse_value(cls, value): + raise NotImplementedError + + def render(self): + """ + Returns the ``struct`` format and list of the size and value. + + The format is derived from the size primitive and the length of the + resulting encoded value (e.g. the format for a string of 'foo' ends + up as 'h3s'. + + .. note :: + The value is expected to be string-able (wrapped in ``str()``) and is + then encoded as UTF-8. + """ + size_format = self.size_primitive.format + + if self.value is None: + return size_format, [-1] + + value = self.render_value(self.value) + + size = len(value) + + format = "%s%ds" % (size_format, size) + + return format, [size, value] + + @classmethod + def parse(cls, buff, offset): + """ + Given a buffer and offset, returns the parsed value and new offset. + + Parses the ``size_primitive`` first to determine how many more bytes to + consume to extract the value. + """ + size, offset = cls.size_primitive.parse(buff, offset) + if size == -1: + return None, offset + + var_struct = struct.Struct("!%ds" % size) + + value = var_struct.unpack_from(buff, offset)[0] + value = cls.parse_value(value) + offset += var_struct.size + + return value, offset + + +class Bool(Primitive): + """ + Represents a boolean (true or false) value. + + Renders as an unsigned char (1 byte). + """ + format = "?" + + +class Byte(Primitive): + """ + Represents a single 8-bit byte. + """ + format = "b" + + +class Int(Primitive): + """ + Represents an 32-bit signed integer. + """ + format = "i" + + +class Long(Primitive): + """ + Represents an 64-bit signed integer. + """ + format = "q" + + +class Float(Primitive): + """ + Represents a single-precision floating poing conforming to IEEE 754. + """ + format = "f" + + +class Double(Primitive): + """ + Represents a double-precision floating poing conforming to IEEE 754. + """ + format = "d" + + +class UString(VariablePrimitive): + """ + Represents a unicode string value, length denoted by a 32-bit integer. + """ + size_primitive = Int + + def render_value(self, value): + return bytes(str(value).encode("utf-8")) + + @classmethod + def parse_value(cls, value): + return value.decode("utf-8") + + def __unicode__(self): + return self.value + + +class Buffer(VariablePrimitive): + """ + Represents a bytestring value, length denoted by a 32-bit signed integer. + """ + size_primitive = Int + + def render_value(self, value): + return bytes(value) + + @classmethod + def parse_value(cls, value): + return value + + +class Vector(Primitive): + """ + Represents an array of any arbitrary `Primitive` or ``Part``. + + Not used directly but rather by its ``of()`` classmethod to denote an + ``Vector.of()``. + """ + item_class = None + + @classmethod + def of(cls, part_class): + """ + Creates a new class with the ``item_class`` attribute properly set. + """ + copy = type( + "VectorOf%s" % part_class.__name__, + cls.__bases__, dict(cls.__dict__) + ) + copy.item_class = part_class + + return copy + + def render(self): + """ + Creates a composite ``struct`` format and the data to render with it. + + The format and data are prefixed with a 32-bit integer denoting the + number of elements, after which each of the items in the array value + are ``render()``-ed and added to the format and data as well. + """ + value = self.value + if value is None: + value = [] + + format = [Int.format] + data = [len(value)] + + for item_value in value: + if issubclass(self.item_class, Primitive): + item = self.item_class(item_value) + else: + item = item_value + + item_format, item_data = item.render() + format.extend(item_format) + data.extend(item_data) + + return "".join(format), data + + @classmethod + def parse(cls, buff, offset): + """ + Parses a raw buffer at offset and returns the resulting array value. + + Starts off by `parse()`-ing the 32-bit element count, followed by + parsing items out of the buffer "count" times. + """ + count, offset = Int.parse(buff, offset) + + values = [] + for i in range(count): + value, new_offset = cls.item_class.parse(buff, offset) + + values.append(value) + offset = new_offset + + return values, offset + + def __str__(self): + return "%s[%s]" % (self.item_class, ", ".join(map(str, self.value))) diff --git a/zoonado/protocol/reconfig.py b/zoonado/protocol/reconfig.py new file mode 100644 index 0000000..33ddf86 --- /dev/null +++ b/zoonado/protocol/reconfig.py @@ -0,0 +1,27 @@ +from .request import Request +from .response import Response +from .stat import Stat +from .primitives import Long, UString + + +class ReconfigRequest(Request): + """ + """ + opcode = 16 + + parts = ( + ("joining_servers", UString), + ("leaving_servers", UString), + ("new_members", UString), + ("current_config_id", Long), + ) + + +class ReconfigResponse(Response): + """ + """ + opcode = 16 + + parts = ( + ("stat", Stat), + ) diff --git a/zoonado/protocol/request.py b/zoonado/protocol/request.py new file mode 100644 index 0000000..433f031 --- /dev/null +++ b/zoonado/protocol/request.py @@ -0,0 +1,44 @@ +import logging +import struct + +from six import BytesIO + +from .part import Part +from .primitives import Int + + +log = logging.getLogger(__name__) + + +class Request(Part): + """ + Returns a bytesring representation of the request instance. + + # TODO(wglass): specify how xid and type preamble goes in + + Since this is a ``Part`` subclass the rest is a matter of + appending the result of a ``render()`` call. + """ + opcode = None + special_xid = None + writes = False + + def serialize(self, xid=None): + buff = BytesIO() + + formats = [] + data = [] + if xid is not None: + formats.append(Int.format) + data.append(xid) + if self.opcode: + formats.append(Int.format) + data.append(self.opcode) + + payload_format, payload_data = self.render() + formats.append(payload_format) + data.extend(payload_data) + + buff.write(struct.pack("!" + "".join(formats), *data)) + + return buff.getvalue() diff --git a/zoonado/protocol/response.py b/zoonado/protocol/response.py new file mode 100644 index 0000000..23f990c --- /dev/null +++ b/zoonado/protocol/response.py @@ -0,0 +1,39 @@ +from .part import Part + + +response_xref = {} + + +class ResponseMeta(type): + + def __new__(cls, name, bases, attrs): + new_class = super(ResponseMeta, cls).__new__(cls, name, bases, attrs) + + response_xref[new_class.opcode] = new_class + + return new_class + + +class Response(Part): + """ + Base class for all operation response classes. + + A simple class, has only an ``opcode`` attribute expected to be defined by + subclasses, and a `deserialize()` classmethod. + """ + __metaclass__ = ResponseMeta + + opcode = None + + @classmethod + def deserialize(cls, raw_bytes): + """ + Deserializes the given raw bytes into an instance. + + Since this is a subclass of ``Part`` but a top-level one (i.e. no other + subclass of ``Part`` would have a ``Response`` as a part) this merely + has to parse the raw bytes and discard the resulting offset. + """ + instance, _ = cls.parse(raw_bytes, offset=0) + + return instance diff --git a/zoonado/protocol/sasl.py b/zoonado/protocol/sasl.py new file mode 100644 index 0000000..9dbaaf2 --- /dev/null +++ b/zoonado/protocol/sasl.py @@ -0,0 +1,23 @@ +from .request import Request +from .response import Response +from .primitives import Buffer + + +class SASLRequest(Request): + """ + """ + opcode = 102 + + parts = ( + ("token", Buffer) + ) + + +class SASLResponse(Response): + """ + """ + opcode = 102 + + parts = ( + ("token", Buffer) + ) diff --git a/zoonado/protocol/stat.py b/zoonado/protocol/stat.py new file mode 100644 index 0000000..1c82696 --- /dev/null +++ b/zoonado/protocol/stat.py @@ -0,0 +1,36 @@ +from .part import Part +from .primitives import Long, Int + + +class Stat(Part): + """ + """ + parts = ( + ("created_zxid", Long), + ("last_modified_zxid", Long), + ("created", Long), + ("modified", Long), + ("version", Int), + ("child_version", Int), + ("acl_version", Int), + ("ephemeral_owner", Long), + ("data_length", Int), + ("num_children", Int), + ("last_modified_children", Long), + ) + + +class StatPersisted(Part): + """ + """ + parts = ( + ("created_zxid", Long), + ("last_modified_zxid", Long), + ("created", Long), + ("modified", Long), + ("version", Int), + ("child_version", Int), + ("acl_version", Int), + ("ephemeral_owner", Long), + ("last_modified_children", Long), + ) diff --git a/zoonado/protocol/sync.py b/zoonado/protocol/sync.py new file mode 100644 index 0000000..61d1e05 --- /dev/null +++ b/zoonado/protocol/sync.py @@ -0,0 +1,23 @@ +from .request import Request +from .response import Response +from .primitives import UString + + +class SyncRequest(Request): + """ + """ + opcode = 9 + + parts = ( + ("path", UString), + ) + + +class SyncResponse(Response): + """ + """ + opcode = 9 + + parts = ( + ("path", UString), + ) diff --git a/zoonado/protocol/transaction.py b/zoonado/protocol/transaction.py new file mode 100644 index 0000000..cb01528 --- /dev/null +++ b/zoonado/protocol/transaction.py @@ -0,0 +1,109 @@ +import logging +import struct + +from six import BytesIO + +from zoonado import exc + +from .request import Request +from .response import Response, response_xref +from .part import Part +from .primitives import Int, Bool + + +error_struct = struct.Struct("!" + Int.format) + + +log = logging.getLogger(__name__) + + +class MultiHeader(Part): + """ + """ + parts = ( + ("type", Int), + ("done", Bool), + ("error", Int), + ) + + +class TransactionRequest(Request): + """ + """ + opcode = 14 + + def __init__(self, *args, **kwargs): + super(TransactionRequest, self).__init__(*args, **kwargs) + self.requests = [] + + def add(self, request): + self.requests.append(request) + + def serialize(self, xid=None): + buff = BytesIO() + + formats = [] + data = [] + if xid is not None: + formats.append(Int.format) + data.append(xid) + if self.opcode: + formats.append(Int.format) + data.append(self.opcode) + + for request in self.requests: + header = MultiHeader(type=request.opcode, done=False, error=-1) + header_format, header_data = header.render() + formats.append(header_format) + data.extend(header_data) + + payload_format, payload_data = request.render() + formats.append(payload_format) + data.extend(payload_data) + + footer = MultiHeader(type=-1, done=True, error=-1) + footer_format, footer_data = footer.render() + formats.append(footer_format) + data.extend(footer_data) + + buff.write(struct.pack("!" + "".join(formats), *data)) + + return buff.getvalue() + + def __str__(self): + return "Txn[%s]" % ", ".join(map(str, self.requests)) + + +class TransactionResponse(Response): + """ + """ + opcode = 14 + + def __init__(self, *args, **kwargs): + super(TransactionResponse, self).__init__(*args, **kwargs) + self.responses = [] + + @classmethod + def deserialize(cls, raw_bytes): + instance = cls() + + header, offset = MultiHeader.parse(raw_bytes, 0) + while not header.done: + if header.type == -1: + error_code = error_struct.unpack_from(raw_bytes, offset)[0] + offset += error_struct.size + instance.responses.append(exc.get_response_error(error_code)) + header, offset = MultiHeader.parse(raw_bytes, offset) + continue + + response_class = response_xref[header.type] + + response, offset = response_class.parse(raw_bytes, offset) + instance.responses.append(response) + + header, offset = MultiHeader.parse(raw_bytes, offset) + + return instance + + def __str__(self): + return "Txn[%s]" % ", ".join(map(str, self.responses)) diff --git a/zoonado/protocol/watches.py b/zoonado/protocol/watches.py new file mode 100644 index 0000000..11b0617 --- /dev/null +++ b/zoonado/protocol/watches.py @@ -0,0 +1,88 @@ +from .request import Request +from .response import Response +from .primitives import Int, Long, Vector, UString + + +WATCH_XID = -1 + + +class WatchEvent(Response): + """ + """ + + CREATED = 1 + DELETED = 2 + DATA_CHANGED = 3 + CHILDREN_CHANGED = 4 + + DISCONNECTED = 0 + CONNECTED = 3 + AUTH_FAILED = 4 + CONNECTED_READ_ONLY = 5 + SASL_AUTHENTICATED = 6 + SESSION_EXPIRED = -112 + + parts = ( + ("type", Int), + ("state", Int), + ("path", UString), + ) + + +class SetWatchesRequest(Request): + """ + """ + opcode = 101 + + parts = ( + ("relative_zxid", Long), + ("data_watches", Vector.of(UString)), + ("exist_watches", Vector.of(UString)), + ("child_watches", Vector.of(UString)), + ) + + +class SetWatchesResponse(Response): + """ + """ + opcode = 101 + + parts = () + + +class CheckWatchesRequest(Request): + """ + """ + opcode = 17 + + parts = ( + ("path", UString), + ("type", Int), + ) + + +class CheckWatchesResponse(Response): + """ + """ + opcode = 17 + + parts = () + + +class RemoveWatchesRequest(Request): + """ + """ + opcode = 18 + + parts = ( + ("path", UString), + ("type", Int), + ) + + +class RemoveWatchesResponse(Response): + """ + """ + opcode = 18 + + parts = () diff --git a/zoonado/recipes/__init__.py b/zoonado/recipes/__init__.py new file mode 100644 index 0000000..85f869d --- /dev/null +++ b/zoonado/recipes/__init__.py @@ -0,0 +1,2 @@ +from .recipe import Recipe # noqa +from .proxy import RecipeProxy # noqa diff --git a/zoonado/recipes/allocator.py b/zoonado/recipes/allocator.py new file mode 100644 index 0000000..41cd252 --- /dev/null +++ b/zoonado/recipes/allocator.py @@ -0,0 +1,145 @@ +import collections +import itertools +import json + +from tornado import gen, ioloop + +from .data_watcher import DataWatcher +from .party import Party +from .lock import Lock +from .recipe import Recipe + + +class Allocator(Recipe): + + sub_recipes = { + "party": (Party, ["member_path", "name"]), + "lock": (Lock, ["lock_path"]), + "data_watcher": DataWatcher, + } + + def __init__(self, base_path, name, allocator_fn=None): + self.name = name + + super(Allocator, self).__init__(base_path) + + if allocator_fn is None: + allocator_fn = round_robin + + self.allocator_fn = allocator_fn + + self.active = False + + self.full_allocation = collections.defaultdict(set) + self.full_set = set() + + @property + def lock_path(self): + return self.base_path + "/lock" + + @property + def member_path(self): + return self.base_path + "/members" + + @property + def allocation(self): + return self.full_allocation[self.name] + + def validate(self, new_allocation): + as_list = [] + for subset in new_allocation.values(): + as_list.extend(list(subset)) + + # make sure there are no duplicates among the subsets + assert len(as_list) == len(set(as_list)), ( + "duplicate items found in allocation: %s" % self.full_allocation + ) + # make sure there's no mismatch beween the full set and allocations + assert len(self.full_set.symmetric_difference(set(as_list))) == 0, ( + "mismatch between full set and allocation: %s vs %s" % ( + self.full_set, self.full_allocation + ) + ) + + @gen.coroutine + def start(self): + self.party.client = self.client + self.party.watcher.client = self.client + self.data_watcher.client = self.client + self.lock.client = self.client + + self.active = True + + yield self.ensure_path() + + yield self.party.join() + + self.data_watcher.add_callback(self.base_path, self.handle_data_change) + + ioloop.IOLoop.current().add_callback(self.monitor_member_changes) + + @gen.coroutine + def add(self, new_item): + new_set = self.full_set.copy().add(new_item) + yield self.update_set(new_set) + + @gen.coroutine + def remove(self, new_item): + new_set = self.full_set.copy().remove(new_item) + yield self.update_set(new_set) + + @gen.coroutine + def update(self, new_items): + new_items = set(new_items) + data = json.dumps(list(new_items)) + + with (yield self.lock.acquire()): + yield self.client.set_data(self.base_path, data=data) + + def monitor_member_changes(self): + while self.active: + yield self.party.wait_for_change() + if not self.active: + break + + self.allocate() + + def handle_data_change(self, new_set_data): + if new_set_data is None: + return + + new_set_data = set(json.loads(new_set_data)) + if new_set_data == self.full_set: + return + + self.full_set = new_set_data + self.allocate() + + def allocate(self): + new_allocation = self.allocator_fn(self.party.members, self.full_set) + self.validate(new_allocation) + self.full_allocation = new_allocation + + @gen.coroutine + def stop(self): + yield self.party.leave() + + self.data_watcher.remove_callback( + self.base_path, self.handle_data_change + ) + + +def round_robin(members, items): + """ + Default allocator with a round robin approach. + + In this algorithm, each member of the group is cycled over and given an + item until there are no items left. This assumes roughly equal capacity + for each member and aims for even distribution of item counts. + """ + allocation = collections.defaultdict(set) + + for member, item in zip(itertools.cycle(members), items): + allocation[member].add(item) + + return allocation diff --git a/zoonado/recipes/barrier.py b/zoonado/recipes/barrier.py new file mode 100644 index 0000000..8ac82d0 --- /dev/null +++ b/zoonado/recipes/barrier.py @@ -0,0 +1,47 @@ +import time + +from tornado import gen + +from zoonado import exc, WatchEvent + +from .recipe import Recipe + + +class Barrier(Recipe): + + def __init__(self, path): + super(Barrier, self).__init__() + self.path = path + + @gen.coroutine + def create(self): + yield self.ensure_path() + + @gen.coroutine + def lift(self): + try: + self.client.delete(self.path) + except exc.NoNode: + pass + + @gen.coroutine + def wait(self, timeout=None): + time_limit = None + if timeout is not None: + time_limit = time.time() + timeout + + barrier_lifted = self.client.wait_for_event( + WatchEvent.DELETED, self.path + ) + + if time_limit: + barrier_lifted = gen.with_timeout(barrier_lifted, time_limit) + + exists = yield self.client.exists(path=self.path, watch=True) + if not exists: + return + + try: + yield barrier_lifted + except gen.TimeoutError: + raise exc.TimeoutError diff --git a/zoonado/recipes/base_lock.py b/zoonado/recipes/base_lock.py new file mode 100644 index 0000000..802b6ea --- /dev/null +++ b/zoonado/recipes/base_lock.py @@ -0,0 +1,83 @@ +import contextlib +import logging +import time + +from tornado import gen, ioloop + +from zoonado import exc, states + +from .sequential import SequentialRecipe + + +log = logging.getLogger(__name__) + + +class BaseLock(SequentialRecipe): + + @gen.coroutine + def wait_in_line(self, znode_label, timeout=None, blocked_by=None): + time_limit = None + if timeout is not None: + time_limit = time.time() + timeout + + yield self.create_unique_znode(znode_label) + + while True: + if time_limit and time.time() >= time_limit: + raise exc.TimeoutError + + owned_positions, contenders = yield self.analyze_siblings() + if znode_label not in owned_positions: + raise exc.SessionLost + + blockers = contenders[:owned_positions[znode_label]] + if blocked_by: + blockers = [ + contender for contender in blockers + if self.determine_znode_label(contender) in blocked_by + ] + + if not blockers: + break + + yield self.wait_on_sibling(blockers[-1], time_limit) + + raise gen.Return(self.make_contextmanager(znode_label)) + + def make_contextmanager(self, znode_label): + state = {"acquired": True} + + def still_acquired(): + return state["acquired"] + + @gen.coroutine + def handle_session_loss(): + yield self.client.session.state.wait_for(states.States.LOST) + if not state["acquired"]: + return + + log.warn( + "Session expired at some point, lock %s no longer acquired.", + self + ) + state["acquired"] = False + + ioloop.IOLoop.current().add_callback(handle_session_loss) + + @gen.coroutine + def on_exit(): + state["acquired"] = False + yield self.delete_unique_znode(znode_label) + + @contextlib.contextmanager + def context_manager(): + try: + yield still_acquired + finally: + ioloop.IOLoop.current().add_callback(on_exit) + + return context_manager() + + +class LockLostError(exc.ZKError): + pass diff --git a/zoonado/recipes/base_watcher.py b/zoonado/recipes/base_watcher.py new file mode 100644 index 0000000..87f8adb --- /dev/null +++ b/zoonado/recipes/base_watcher.py @@ -0,0 +1,49 @@ +import collections +import logging + +from tornado import ioloop, gen + +from zoonado import exc + +from .recipe import Recipe + + +log = logging.getLogger(__name__) + + +class BaseWatcher(Recipe): + + watched_event = None + + def __init__(self, *args, **kwargs): + super(BaseWatcher, self).__init__(*args, **kwargs) + self.callbacks = collections.defaultdict(set) + + def add_callback(self, path, callback): + self.callbacks[path].add(callback) + + if len(self.callbacks[path]) == 1: + ioloop.IOLoop.current().add_callback(self.watch_loop, path) + + def remove_callback(self, path, callback): + self.callbacks[path].discard(callback) + + @gen.coroutine + def fetch(self, path): + raise NotImplementedError + + @gen.coroutine + def watch_loop(self, path): + while self.callbacks[path]: + wait = self.client.wait_for_event(self.watched_event, path) + + log.debug("Fetching data for %s", path) + try: + result = yield self.fetch(path) + except exc.NoNode: + return + + yield wait + + for callback in self.callbacks[path]: + ioloop.IOLoop.current().add_callback(callback, result) diff --git a/zoonado/recipes/children_watcher.py b/zoonado/recipes/children_watcher.py new file mode 100644 index 0000000..69c77e0 --- /dev/null +++ b/zoonado/recipes/children_watcher.py @@ -0,0 +1,14 @@ +from tornado import gen +from zoonado import WatchEvent + +from .base_watcher import BaseWatcher + + +class ChildrenWatcher(BaseWatcher): + + watched_event = WatchEvent.CHILDREN_CHANGED + + @gen.coroutine + def fetch(self, path): + children = yield self.client.get_children(path=path, watch=True) + raise gen.Return(children) diff --git a/zoonado/recipes/counter.py b/zoonado/recipes/counter.py new file mode 100644 index 0000000..f00738e --- /dev/null +++ b/zoonado/recipes/counter.py @@ -0,0 +1,90 @@ +import logging + +from tornado import gen, concurrent + +from zoonado import exc + +from .data_watcher import DataWatcher +from .recipe import Recipe + + +log = logging.getLogger(__name__) + + +class Counter(Recipe): + + sub_recipes = { + "watcher": DataWatcher + } + + def __init__(self, base_path, use_float=False): + super(Counter, self).__init__(base_path) + + self.value = None + + if use_float: + self.numeric_type = float + else: + self.numeric_type = int + + self.value_sync = concurrent.Future() + + @gen.coroutine + def start(self): + self.watcher.add_callback(self.base_path, self.data_callback) + yield gen.moment + + yield self.ensure_path() + + raw_value = yield self.client.get_data(self.base_path) + self.value = self.numeric_type(raw_value or 0) + + def data_callback(self, new_value): + self.value = self.numeric_type(new_value) + if not self.value_sync.done(): + self.value_sync.set_result(None) + self.value_sync = concurrent.Future() + + @gen.coroutine + def set_value(self, value, force=True): + data = str(value) + yield self.client.set_data(self.base_path, data, force=force) + log.debug("Set value to '%s': successful", data) + yield self.value_sync + + @gen.coroutine + def apply_operation(self, operation): + success = False + while not success: + data = str(operation(self.value)) + try: + yield self.client.set_data(self.base_path, data, force=False) + log.debug("Operation '%s': successful", operation.__name__) + yield self.value_sync + success = True + except exc.BadVersion: + log.debug( + "Operation '%s': version mismatch, retrying", + operation.__name__ + ) + yield self.value_sync + + @gen.coroutine + def incr(self): + + def increment(value): + return value + 1 + + yield self.apply_operation(increment) + + @gen.coroutine + def decr(self): + + def decrement(value): + return value - 1 + + yield self.apply_operation(decrement) + + @gen.coroutine + def stop(self): + self.watcher.remove_callback(self.base_path, self.data_callback) diff --git a/zoonado/recipes/data_watcher.py b/zoonado/recipes/data_watcher.py new file mode 100644 index 0000000..15cb0f1 --- /dev/null +++ b/zoonado/recipes/data_watcher.py @@ -0,0 +1,14 @@ +from tornado import gen +from zoonado import WatchEvent + +from .base_watcher import BaseWatcher + + +class DataWatcher(BaseWatcher): + + watched_event = WatchEvent.DATA_CHANGED + + @gen.coroutine + def fetch(self, path): + data = yield self.client.get_data(path=path, watch=True) + raise gen.Return(data) diff --git a/zoonado/recipes/double_barrier.py b/zoonado/recipes/double_barrier.py new file mode 100644 index 0000000..4e0bfac --- /dev/null +++ b/zoonado/recipes/double_barrier.py @@ -0,0 +1,76 @@ +import logging +import time + +from tornado import gen + +from zoonado import exc, WatchEvent + +from .sequential import SequentialRecipe + +log = logging.getLogger(__name__) + + +class DoubleBarrier(SequentialRecipe): + + def __init__(self, base_path, min_participants): + super(DoubleBarrier, self).__init__(base_path) + self.min_participants = min_participants + + @property + def sentinel_path(self): + return self.sibling_path("sentinel") + + @gen.coroutine + def enter(self, timeout=None): + log.debug("Entering double barrier %s", self.base_path) + time_limit = None + if timeout is not None: + time_limit = time.time() + timeout + + barrier_lifted = self.client.wait_for_event( + WatchEvent.CREATED, self.sentinel_path + ) + if time_limit: + barrier_lifted = gen.with_timeout(barrier_lifted, time_limit) + + exists = yield self.client.exists(path=self.sentinel_path, watch=True) + + yield self.create_unique_znode("worker") + + owned_positions, participants = yield self.analyze_siblings() + + if exists: + return + + elif len(participants) >= self.min_participants: + yield self.create_znode(self.sentinel_path) + return + + try: + yield barrier_lifted + except gen.TimeoutError: + raise exc.TimeoutError + + @gen.coroutine + def leave(self, timeout=None): + log.debug("Leaving double barrier %s", self.base_path) + time_limit = None + if timeout is not None: + time_limit = time.time() + timeout + + owned_positions, participants = yield self.analyze_siblings() + while len(participants) > 1: + if owned_positions["worker"] == 0: + yield self.wait_on_sibling(participants[-1], time_limit) + else: + yield self.delete_unique_znode("worker") + yield self.wait_on_sibling(participants[0], time_limit) + + owned_positions, participants = yield self.analyze_siblings() + + if len(participants) == 1 and "worker" in owned_positions: + yield self.delete_unique_znode("worker") + try: + yield self.client.delete(self.sentinel_path) + except exc.NoNode: + pass diff --git a/zoonado/recipes/election.py b/zoonado/recipes/election.py new file mode 100644 index 0000000..ad535f6 --- /dev/null +++ b/zoonado/recipes/election.py @@ -0,0 +1,55 @@ +import time + +from tornado import gen, ioloop, concurrent + +from .sequential import SequentialRecipe + + +class LeaderElection(SequentialRecipe): + + def __init__(self, base_path): + super(LeaderElection, self).__init__(base_path) + self.has_leadership = False + + self.leadership_future = concurrent.Future() + + @gen.coroutine + def join(self): + yield self.create_unique_znode("candidate") + yield self.check_position() + + @gen.coroutine + def check_position(self, _=None): + owned_positions, candidates = yield self.analyze_siblings() + if "candidate" not in owned_positions: + return + + position = owned_positions["candidate"] + + self.has_leadership = bool(position == 0) + + if self.has_leadership: + self.leadership_future.set_result(None) + return + + moved_up = self.wait_on_sibling(candidates[position - 1]) + + ioloop.IOLoop.current().add_future(moved_up, self.check_position) + + @gen.coroutine + def wait_for_leadership(self, timeout=None): + if self.has_leadership: + return + + time_limit = None + if timeout is not None: + time_limit = time.time() + timeout + + if time_limit: + yield gen.with_timeout(self.leadership_future, time_limit) + else: + yield self.leadership_future + + @gen.coroutine + def resign(self): + yield self.delete_unique_znode("candidate") diff --git a/zoonado/recipes/lease.py b/zoonado/recipes/lease.py new file mode 100644 index 0000000..a37f41c --- /dev/null +++ b/zoonado/recipes/lease.py @@ -0,0 +1,39 @@ +import logging +import time + +from tornado import gen, ioloop +from zoonado import exc + +from .sequential import SequentialRecipe + + +log = logging.getLogger(__name__) + + +class Lease(SequentialRecipe): + + def __init__(self, base_path, limit=1): + super(Lease, self).__init__(base_path) + self.limit = limit + + @gen.coroutine + def obtain(self, duration): + lessees = yield self.client.get_children(self.base_path) + + if len(lessees) >= self.limit: + raise gen.Return(False) + + time_limit = time.time() + duration.total_seconds() + + try: + yield self.create_unique_znode("lease", data=str(time_limit)) + except exc.NodeExists: + log.warn("Lease for %s already obtained.", self.base_path) + + ioloop.IOLoop.current().call_at(time_limit, self.release) + + raise gen.Return(True) + + @gen.coroutine + def release(self): + yield self.delete_unique_znode("lease") diff --git a/zoonado/recipes/lock.py b/zoonado/recipes/lock.py new file mode 100644 index 0000000..28501ae --- /dev/null +++ b/zoonado/recipes/lock.py @@ -0,0 +1,19 @@ +from tornado import gen + +from zoonado import exc + +from .base_lock import BaseLock + + +class Lock(BaseLock): + + @gen.coroutine + def acquire(self, timeout=None): + result = None + while not result: + try: + result = yield self.wait_in_line("lock", timeout) + except exc.SessionLost: + continue + + raise gen.Return(result) diff --git a/zoonado/recipes/party.py b/zoonado/recipes/party.py new file mode 100644 index 0000000..ae71ec7 --- /dev/null +++ b/zoonado/recipes/party.py @@ -0,0 +1,46 @@ +from tornado import gen, concurrent + +from .children_watcher import ChildrenWatcher +from .sequential import SequentialRecipe + + +class Party(SequentialRecipe): + + sub_recipes = { + "watcher": ChildrenWatcher, + } + + def __init__(self, base_path, name): + super(Party, self).__init__(base_path) + + self.name = name + self.members = [] + self.change_future = None + + @gen.coroutine + def join(self): + yield self.create_unique_znode(self.name) + yield self.analyze_siblings() + self.watcher.add_callback(self.base_path, self.update_members) + + @gen.coroutine + def wait_for_change(self): + if not self.change_future or self.change_future.done(): + self.change_future = concurrent.Future() + + yield self.change_future + + @gen.coroutine + def leave(self): + self.watcher.remove_callback(self.base_path, self.update_members) + yield self.delete_unique_znode(self.name) + + def update_members(self, raw_sibling_names): + new_members = [ + self.determine_znode_label(sibling) + for sibling in raw_sibling_names + ] + + self.members = new_members + if self.change_future and not self.change_future.done(): + self.change_future.set_result(new_members) diff --git a/zoonado/recipes/proxy.py b/zoonado/recipes/proxy.py new file mode 100644 index 0000000..b591d3d --- /dev/null +++ b/zoonado/recipes/proxy.py @@ -0,0 +1,63 @@ +import logging +import pkg_resources + +from .recipe import Recipe + + +ENTRY_POINT = "zoonado.recipes" + +log = logging.getLogger(__name__) + + +class RecipeClassProxy(object): + + def __init__(self, client, recipe_class): + self.client = client + self.recipe_class = recipe_class + + def __call__(self, *args, **kwargs): + recipe = self.recipe_class(*args, **kwargs) + recipe.set_client(self.client) + return recipe + + +class RecipeProxy(object): + + def __init__(self, client): + self.client = client + + self.installed_classes = {} + self.gather_installed_classes() + + def __getattr__(self, name): + if name not in self.installed_classes: + raise AttributeError("No such recipe: %s" % name) + + return RecipeClassProxy(self.client, self.installed_classes[name]) + + def gather_installed_classes(self): + for entry_point in pkg_resources.iter_entry_points(ENTRY_POINT): + try: + recipe_class = entry_point.load() + except ImportError as e: + log.error( + "Could not load recipe %s: %s", entry_point.name, str(e) + ) + continue + + if not issubclass(recipe_class, Recipe): + log.error( + "Could not load recipe %s: not a Recipe subclass", + entry_point.name + ) + continue + + if not recipe_class.validate_dependencies(): + log.error( + "Could not load recipe %s: %s has unmet dependencies", + entry_point.name, recipe_class.__name__ + ) + continue + + log.debug("Loaded recipe %s", recipe_class.__name__) + self.installed_classes[recipe_class.__name__] = recipe_class diff --git a/zoonado/recipes/recipe.py b/zoonado/recipes/recipe.py new file mode 100644 index 0000000..6b9ad30 --- /dev/null +++ b/zoonado/recipes/recipe.py @@ -0,0 +1,48 @@ +from tornado import gen + +from zoonado import exc + + +class Recipe(object): + + sub_recipes = {} + + def __init__(self, base_path="/"): + self.client = None + self.base_path = base_path + + for attribute_name, recipe_class in self.sub_recipes.items(): + recipe_args = ["base_path"] + if isinstance(recipe_class, tuple): + recipe_class, recipe_args = recipe_class + + recipe_args = [getattr(self, arg) for arg in recipe_args] + + recipe = recipe_class(*recipe_args) + + setattr(self, attribute_name, recipe) + + def set_client(self, client): + self.client = client + for sub_recipe in self.sub_recipes: + getattr(self, sub_recipe).client = client + + @classmethod + def validate_dependencies(cls): + return True + + @gen.coroutine + def ensure_path(self): + yield self.client.ensure_path(self.base_path) + + @gen.coroutine + def create_znode(self, path): + try: + yield self.client.create(path) + except exc.NodeExists: + pass + except exc.NoNode: + try: + yield self.ensure_path() + except exc.NodeExists: + pass diff --git a/zoonado/recipes/sequential.py b/zoonado/recipes/sequential.py new file mode 100644 index 0000000..1fa732c --- /dev/null +++ b/zoonado/recipes/sequential.py @@ -0,0 +1,88 @@ +import logging +import re +import uuid + +from tornado import gen +from zoonado import exc, WatchEvent + +from .recipe import Recipe + + +log = logging.getLogger(__name__) + +sequential_re = re.compile(r'.*[0-9]{10}$') + + +class SequentialRecipe(Recipe): + + def __init__(self, base_path): + super(SequentialRecipe, self).__init__(base_path) + self.guid = uuid.uuid4().hex + + self.owned_paths = {} + + def sequence_number(self, sibling): + return int(sibling[-10:]) + + def determine_znode_label(self, sibling): + return sibling.rsplit("-", 2)[0] + + def sibling_path(self, path): + return "/".join([self.base_path, path]) + + @gen.coroutine + def create_unique_znode(self, znode_label, data=None): + path = self.sibling_path(znode_label + "-" + self.guid + "-") + + try: + created_path = yield self.client.create( + path, data=data, ephemeral=True, sequential=True + ) + except exc.NoNode: + yield self.ensure_path() + created_path = yield self.client.create( + path, data=data, ephemeral=True, sequential=True + ) + + self.owned_paths[znode_label] = created_path + + @gen.coroutine + def delete_unique_znode(self, znode_label): + try: + yield self.client.delete(self.owned_paths[znode_label]) + except exc.NoNode: + pass + + @gen.coroutine + def analyze_siblings(self): + siblings = yield self.client.get_children(self.base_path) + siblings = [name for name in siblings if sequential_re.match(name)] + + siblings.sort(key=self.sequence_number) + + owned_positions = {} + + for index, path in enumerate(siblings): + if self.guid in path: + owned_positions[self.determine_znode_label(path)] = index + + raise gen.Return((owned_positions, siblings)) + + @gen.coroutine + def wait_on_sibling(self, sibling, time_limit=None): + log.debug("Waiting on sibling %s", sibling) + + path = self.sibling_path(sibling) + + unblocked = self.client.wait_for_event(WatchEvent.DELETED, path) + if time_limit: + unblocked = gen.with_timeout(unblocked, time_limit) + + exists = yield self.client.exists(path=path, watch=True) + if not exists: + unblocked.set_result(None) + + try: + yield unblocked + except gen.TimeoutError: + raise exc.TimeoutError diff --git a/zoonado/recipes/shared_lock.py b/zoonado/recipes/shared_lock.py new file mode 100644 index 0000000..24e6842 --- /dev/null +++ b/zoonado/recipes/shared_lock.py @@ -0,0 +1,32 @@ +from tornado import gen + +from zoonado import exc + +from .base_lock import BaseLock + + +class SharedLock(BaseLock): + + @gen.coroutine + def acquire_read(self, timeout=None): + result = None + while not result: + try: + result = yield self.wait_in_line( + "read", timeout, blocked_by=("write") + ) + except exc.SessionLost: + continue + + raise gen.Return(result) + + @gen.coroutine + def acquire_write(self, timeout=None): + result = None + while not result: + try: + result = yield self.wait_in_line("write", timeout) + except exc.SessionLost: + continue + + raise gen.Return(result) diff --git a/zoonado/recipes/tree_cache.py b/zoonado/recipes/tree_cache.py new file mode 100644 index 0000000..18d7c1e --- /dev/null +++ b/zoonado/recipes/tree_cache.py @@ -0,0 +1,128 @@ +import logging + +import six +from tornado import gen, ioloop + +from .children_watcher import ChildrenWatcher +from .data_watcher import DataWatcher +from .recipe import Recipe + + +log = logging.getLogger(__name__) + + +class TreeCache(Recipe): + + sub_recipes = { + "data_watcher": DataWatcher, + "child_watcher": ChildrenWatcher, + } + + def __init__(self, base_path, defaults=None): + super(TreeCache, self).__init__(base_path) + self.defaults = defaults or {} + self.root = None + + @gen.coroutine + def start(self): + log.debug("Starting znode tree cache at %s", self.base_path) + + self.root = ZNodeCache( + self.base_path, self.defaults, + self.client, self.data_watcher, self.children_watcher, + ) + + yield self.ensure_path() + + yield self.root.start() + + def stop(self): + self.root.stop() + + def __getattr__(self, attribute): + return getattr(self.root, attribute) + + def as_dict(self): + return self.root.as_dict() + + +class ZNodeCache(object): + + def __init__(self, path, defaults, client, data_watcher, child_watcher): + self.path = path + + self.client = client + self.defaults = defaults + + self.data_watcher = data_watcher + self.child_watcher = child_watcher + + self.children = {} + self.data = None + + @property + def dot_path(self): + return self.path[1:].replace("/", ".") + + @property + def value(self): + return self.data + + def __getattr__(self, name): + if name not in self.children: + raise AttributeError + + return self.children[name] + + @gen.coroutine + def start(self): + data, children = yield [ + self.client.get_data(self.path), + self.client.get_children(self.path) + ] + + self.data = data + for child in children: + self.children[child] = ZNodeCache( + self.path + "/" + child, self.defaults.get(child, {}), + self.client, self.data_watcher, self.child_watcher + ) + + yield [child.start() for child in self.children.values()] + + self.data_watcher.add_callback(self.path, self.data_callback) + self.child_watcher.add_callback(self.path, self.child_callback) + + def stop(self): + self.data_watcher.remove_callback(self.path, self.data_callback) + self.child_watcher.remove_callback(self.path, self.child_callback) + + def child_callback(self, new_children): + removed_children = set(self.children.keys()) - set(new_children) + added_children = set(new_children) - set(self.children.keys()) + + for removed in removed_children: + log.debug("Removed child %s", self.dot_path + "." + removed) + child = self.children.pop(removed) + child.stop() + + for added in added_children: + log.debug("added child %s", self.dot_path + "." + removed) + self.children[added] = ZNodeCache( + self.path + "/" + added, + self.client, self.data_watcher, self.child_watcher + ) + ioloop.IOLoop.current.add_callback(self.children[added].start) + + def data_callback(self, data): + log.debug("New value for %s: %r", self.dot_path, data) + self.data = data + + def as_dict(self): + if self.children: + return { + child_path: child_znode.as_dict() + for child_path, child_znode in six.iteritems(self.children) + } + + return self.data diff --git a/zoonado/retry.py b/zoonado/retry.py new file mode 100644 index 0000000..553e803 --- /dev/null +++ b/zoonado/retry.py @@ -0,0 +1,82 @@ +import collections +import logging +import time + +from tornado import gen + +from zoonado import exc + + +log = logging.getLogger(__name__) + + +class RetryPolicy(object): + + def __init__(self, try_limit, sleep_func): + self.try_limit = try_limit + self.sleep_func = sleep_func + + self.timings = collections.defaultdict(list) + + @gen.coroutine + def enforce(self, request=None): + self.timings[id(request)].append(time.time()) + + tries = len(self.timings[id(request)]) + if tries == 1: + return + + if self.try_limit is not None and tries >= self.try_limit: + raise exc.FailedRetry + + wait_time = self.sleep_func(self.timings[id(request)]) + if wait_time is None or wait_time == 0: + return + elif wait_time < 0: + raise exc.FailedRetry + + log.debug("Waiting %d seconds until next try.", wait_time) + yield gen.sleep(wait_time) + + def clear(self, request): + self.timings.pop(id(request), None) + + @classmethod + def once(cls): + return cls.n_times(1) + + @classmethod + def n_times(cls, n): + + def never_wait(timings): + return None + + return cls(try_limit=None, sleep_func=never_wait) + + @classmethod + def forever(cls): + + def never_wait(timings): + return None + + return cls(try_limit=None, sleep_func=never_wait) + + @classmethod + def exponential_backoff(cls, base=2, maximum=None): + + def exponential(timings): + wait_time = base ** len(timings) + if maximum is not None: + wait_time = max(maximum, wait_time) + + return wait_time + + return cls(try_limit=None, sleep_func=exponential) + + @classmethod + def until_elapsed(cls, timeout): + + def elapsed_time(timings): + return time.time() - timings[0] + + return cls(try_limit=None, sleep_func=elapsed_time) diff --git a/zoonado/session.py b/zoonado/session.py new file mode 100644 index 0000000..19844ea --- /dev/null +++ b/zoonado/session.py @@ -0,0 +1,274 @@ +import collections +import logging +import random + +from tornado import gen, ioloop + +from zoonado import protocol, exc +from .connection import Connection +from .states import States, SessionStateMachine +from .retry import RetryPolicy + + +DEFAULT_ZOOKEEPER_PORT = 2181 + +MAX_FIND_WAIT = 60 # in seconds + + +log = logging.getLogger(__name__) + + +class Session(object): + + def __init__(self, servers, timeout, retry_policy, allow_read_only): + self.hosts = [] + for server in servers.split(","): + if ":" in server: + host, port = server.split(":") + else: + host = server + port = DEFAULT_ZOOKEEPER_PORT + + self.hosts.append((host, port)) + + self.conn = None + self.state = SessionStateMachine() + + self.retry_policy = retry_policy or RetryPolicy.forever() + self.allow_read_only = allow_read_only + + self.xid = 0 + self.last_zxid = None + + self.session_id = None + self.timeout = timeout + self.password = b'\x00' + + self.heartbeat_handle = None + + self.watch_callbacks = collections.defaultdict(set) + + self.closing = False + + @gen.coroutine + def ensure_safe_state(self, write=False): + safe_states = [States.CONNECTED] + if self.allow_read_only and not write: + safe_states.append(States.READ_ONLY) + + if self.state in safe_states: + return + + yield self.state.wait_for(*safe_states) + + @gen.coroutine + def start(self): + io_loop = ioloop.IOLoop.current() + io_loop.add_callback(self.set_heartbeat) + io_loop.add_callback(self.repair_loop) + + yield self.ensure_safe_state() + + @gen.coroutine + def find_server(self, allow_read_only): + conn = None + + retry_policy = RetryPolicy.exponential_backoff(maximum=MAX_FIND_WAIT) + + while not conn: + yield retry_policy.enforce() + + servers = random.sample(self.hosts, len(self.hosts)) + for host, port in servers: + log.info("Connecting to %s:%s", host, port) + conn = yield self.make_connection(host, port) + if not conn or (conn.start_read_only and not allow_read_only): + continue + + if not conn: + log.warn("No servers available, will keep trying.") + + old_conn = self.conn + self.conn = conn + + if old_conn: + ioloop.IOLoop.current().add_callback(old_conn.close, self.timeout) + + if conn.start_read_only: + ioloop.IOLoop.current().add_callback(self.find_server, False) + + @gen.coroutine + def make_connection(self, host, port): + conn = Connection(host, port, watch_handler=self.event_dispatch) + try: + yield conn.connect() + except Exception: + log.exception("Couldn't connect to %s:%s", host, port) + return + + raise gen.Return(conn) + + @gen.coroutine + def establish_session(self): + log.info("Establising session.") + zxid, response = yield self.conn.send_connect( + protocol.ConnectRequest( + protocol_version=0, + last_seen_zxid=self.last_zxid or 0, + timeout=int((self.timeout or 0) * 1000), + session_id=self.session_id or 0, + password=self.password, + read_only=self.allow_read_only, + ) + ) + self.last_zxid = zxid + + if response.session_id == 0: # invalid session, probably expired + self.state.transition_to(States.LOST) + raise exc.SessionLost() + + log.info("Got session id %s", hex(response.session_id)) + log.info("Negotiated timeout: %s seconds", response.timeout / 1000) + + self.session_id = response.session_id + self.password = response.password + self.timeout = response.timeout / 1000 + + self.last_zxid = zxid + + @gen.coroutine + def repair_loop(self): + while not self.closing: + yield self.state.wait_for(States.SUSPENDED, States.LOST) + if self.closing: + break + + yield self.find_server(allow_read_only=self.allow_read_only) + + session_was_lost = self.state == States.LOST + + try: + yield self.establish_session() + except exc.SessionLost: + self.conn.abort(exc.SessionLost) + yield self.conn.close() + self.session_id = None + self.password = b'\x00' + continue + + if self.conn.start_read_only: + self.state.transition_to(States.READ_ONLY) + else: + self.state.transition_to(States.CONNECTED) + + self.conn.start_read_loop() + + if session_was_lost: + yield self.set_existing_watches() + + @gen.coroutine + def send(self, request): + response = None + while not response: + yield self.retry_policy.enforce(request) + yield self.ensure_safe_state(write=request.writes) + + try: + self.xid += 1 + zxid, response = yield self.conn.send(request, xid=self.xid) + self.last_zxid = zxid + self.set_heartbeat() + self.retry_policy.clear(request) + except exc.ConnectError: + self.state.transition_to(States.SUSPENDED) + + raise gen.Return(response) + + def set_heartbeat(self): + timeout = self.timeout / 3 + + io_loop = ioloop.IOLoop.current() + + if self.heartbeat_handle: + io_loop.remove_timeout(self.heartbeat_handle) + + self.heartbeat_handle = io_loop.call_later(timeout, self.heartbeat) + + @gen.coroutine + def heartbeat(self): + if self.closing: + return + + yield self.ensure_safe_state() + + try: + zxid, _ = yield self.conn.send(protocol.PingRequest()) + self.last_zxid = zxid + except exc.ConnectError: + self.state.transition_to(States.SUSPENDED) + finally: + self.set_heartbeat() + + def add_watch_callback(self, event_type, path, callback): + self.watch_callbacks[(event_type, path)].add(callback) + + def remove_watch_callback(self, event_type, path, callback): + self.watch_callbacks[(event_type, path)].discard(callback) + + def event_dispatch(self, event): + log.debug("Got watch event: %s", event) + + if event.type: + key = (event.type, event.path) + for callback in self.watch_callbacks[key]: + ioloop.IOLoop.current().add_callback(callback, event.path) + return + + if event.state == protocol.WatchEvent.DISCONNECTED: + log.error("Got 'disconnected' watch event.") + self.state.transition_to(States.LOST) + elif event.state == protocol.WatchEvent.SESSION_EXPIRED: + log.error("Got 'session expired' watch event.") + self.state.transition_to(States.LOST) + elif event.state == protocol.WatchEvent.AUTH_FAILED: + log.error("Got 'auth failed' watch event.") + self.state.transition_to(States.LOST) + elif event.state == protocol.WatchEvent.CONNECTED_READ_ONLY: + log.warn("Got 'connected read only' watch event.") + self.state.transition_to(States.READ_ONLY) + elif event.state == protocol.WatchEvent.SASL_AUTHENTICATED: + log.info("Authentication successful.") + elif event.state == protocol.WatchEvent.CONNECTED: + log.info("Got 'connected' watch event.") + self.state.transition_to(States.CONNECTED) + + @gen.coroutine + def set_existing_watches(self): + if not self.watch_callbacks: + return + + request = protocol.SetWatchesRequest( + relative_zxid=self.last_zxid or 0, + data_watches=[], + exist_watches=[], + child_watches=[], + ) + + for event_type, path in self.watch_callbacks.keys(): + if event_type == protocol.WatchEvent.CREATED: + request.exist_watches.append(path) + if event_type == protocol.WatchEvent.DATA_CHANGED: + request.data_watches.append(path) + elif event_type == protocol.WatchEvent.CHILDREN_CHANGED: + request.child_watches.append(path) + + yield self.send(request) + + @gen.coroutine + def close(self): + self.closing = True + + yield self.send(protocol.CloseRequest()) + self.state.transition_to(States.LOST) + + yield self.conn.close(self.timeout) diff --git a/zoonado/states.py b/zoonado/states.py new file mode 100644 index 0000000..ea2c31f --- /dev/null +++ b/zoonado/states.py @@ -0,0 +1,66 @@ +import collections +import logging + +from tornado import concurrent + +from .iterables import drain + + +log = logging.getLogger(__name__) + + +class States(object): + + CONNECTED = "connected" + SUSPENDED = "suspended" + READ_ONLY = "read_only" + LOST = "lost" + + +class SessionStateMachine(object): + + valid_transitions = set([ + (States.LOST, States.CONNECTED), + (States.CONNECTED, States.SUSPENDED), + (States.READ_ONLY, States.CONNECTED), + (States.CONNECTED, States.LOST), + (States.SUSPENDED, States.CONNECTED), + (States.SUSPENDED, States.LOST), + ]) + + def __init__(self): + self.current_state = States.LOST + self.futures = collections.defaultdict(set) + + def transition_to(self, state): + if (self.current_state, state) not in self.valid_transitions: + raise RuntimeError( + "Invalid session state transition: %s -> %s" % ( + self.current_state, state + ) + ) + + log.debug("Session transition: %s -> %s", self.current_state, state) + + self.current_state = state + + for future in drain(self.futures[state]): + if not future.done(): + future.set_result(None) + + def wait_for(self, *states): + f = concurrent.Future() + + if self.current_state in states: + f.set_result(None) + else: + for state in states: + self.futures[state].add(f) + + return f + + def __eq__(self, state): + return self.current_state == state + + def __ne__(self, state): + return self.current_state != state diff --git a/zoonado/transaction.py b/zoonado/transaction.py new file mode 100644 index 0000000..debaea5 --- /dev/null +++ b/zoonado/transaction.py @@ -0,0 +1,87 @@ +from tornado import gen + +from zoonado import protocol + + +class Transaction(object): + + def __init__(self, client): + self.client = client + self.request = protocol.TransactionRequest() + + def check_version(self, path, version): + path = self.client.normalize_path(path) + + self.request.add( + protocol.CheckVersionRequest(path=path, version=version) + ) + + def create( + self, path, data=None, acl=None, + ephemeral=False, sequential=False, container=False + ): + if container and not self.client.features.containers: + raise ValueError("Cannot create container, feature unavailable.") + + path = self.client.normalize_path(path) + acl = acl or self.client.default_acl + + if self.client.features.create_with_stat: + request_class = protocol.Create2Request + else: + request_class = protocol.CreateRequest + + request = request_class(path=path, data=data, acl=acl) + request.set_flags(ephemeral, sequential, container) + + self.request.add(request) + + def set_data(self, path, data, version=-1): + path = self.client.normalize_path(path) + + self.request.add( + protocol.SetDataRequest(path=path, data=data, version=version) + ) + + def delete(self, path, version=-1): + path = self.client.normalize_path(path) + + self.request.add( + protocol.DeleteRequest(path=path, version=version) + ) + + @gen.coroutine + def commit(self): + if not self.request.requests: + raise ValueError("No operations to commit.") + + response = yield self.client.send(self.request) + pairs = zip(self.request.requests, response.responses) + + result = Result() + for request, response in pairs: + if isinstance(response, protocol.CreateResponse): + result.created.add(self.client.denormalize(response.path)) + elif isinstance(response, protocol.SetDataResponse): + result.updated.add(self.client.denormalize(request.path)) + elif isinstance(response, protocol.DeleteResponse): + result.deleted.add(self.client.denormalize(request.path)) + + raise gen.Return(result) + + +class Result(object): + + def __init__(self): + self.checked = set() + self.created = set() + self.updated = set() + self.deleted = set() + + def __nonzero__(self): + return sum([ + len(self.checked), + len(self.created), + len(self.updated), + len(self.deleted), + ]) > 0