From 1d0698016c25f018613828d760b495f150b61655 Mon Sep 17 00:00:00 2001 From: Oleksandr Golovatyi Date: Tue, 20 Aug 2019 13:38:25 -0700 Subject: [PATCH] v0.8: Make tabpy pip package (#332) ### Improvements - TabPy is pip package now - Models are deployed with updated script --- .coveragerc | 9 +- .gitattributes | 1 - .gitignore | 7 +- .scrutinizer.yml | 24 +++ .travis.yml | 7 +- .vscode/launch.json | 16 -- CHANGELOG | 10 +- CONTRIBUTING.md | 81 ++++---- MANIFEST.in | 7 + README.md | 6 +- VERSION | 2 +- docs/FAQ.md | 4 + docs/server-config.md | 178 +++++++++++++----- docs/server-download.md | 13 -- docs/server-install.md | 49 +++++ docs/server-startup.md | 127 ------------- docs/tabpy-tools.md | 55 +++--- docs/tabpy-virtualenv.md | 55 ++---- setup.py | 105 +++++++++++ startup.cmd | 88 +-------- startup.sh | 116 +----------- tabpy-server/setup.py | 51 ----- tabpy-server/tabpy_server/__init__.py | 18 -- .../tabpy_server/handlers/__init__.py | 13 -- tabpy-server/tabpy_server/tabpy.py | 10 - tabpy-tools/setup.py | 30 --- tabpy-tools/tabpy_tools/__init__.py | 23 --- {models/utils => tabpy}/__init__.py | 0 .../app => tabpy/models}/__init__.py | 0 .../setup.py => tabpy/models/deploy_models.py | 8 +- {models => tabpy/models}/scripts/PCA.py | 10 +- .../models}/scripts/SentimentAnalysis.py | 17 +- .../models/scripts}/__init__.py | 0 {models => tabpy/models}/scripts/tTest.py | 10 +- .../models/utils}/__init__.py | 0 {models => tabpy/models}/utils/setup_utils.py | 21 ++- tabpy/tabpy.py | 36 ++++ .../psws => tabpy/tabpy_server}/__init__.py | 0 .../tabpy_server/app/ConfigParameters.py | 0 .../tabpy_server/app/SettingsParameters.py | 0 tabpy/tabpy_server/app/__init__.py | 0 .../tabpy_server/app/app.py | 82 +++++--- .../tabpy_server/app/util.py | 0 tabpy/tabpy_server/common/__init__.py | 0 .../tabpy_server/common/default.conf | 14 +- .../tabpy_server/common/endpoint_file_mgr.py | 0 .../tabpy_server/common/messages.py | 0 .../tabpy_server/common/util.py | 0 tabpy/tabpy_server/handlers/__init__.py | 13 ++ .../tabpy_server/handlers/base_handler.py | 4 +- .../tabpy_server/handlers/endpoint_handler.py | 16 +- .../handlers/endpoints_handler.py | 8 +- .../handlers/evaluation_plane_handler.py | 4 +- .../tabpy_server/handlers/main_handler.py | 2 +- .../handlers/management_handler.py | 10 +- .../handlers/query_plane_handler.py | 6 +- .../handlers/service_info_handler.py | 4 +- .../tabpy_server/handlers/status_handler.py | 2 +- .../handlers/upload_destination_handler.py | 4 +- .../tabpy_server/handlers/util.py | 2 +- tabpy/tabpy_server/management/__init__.py | 0 .../tabpy_server/management/state.py | 2 +- .../tabpy_server/management/util.py | 4 +- tabpy/tabpy_server/psws/__init__.py | 0 .../tabpy_server/psws/callbacks.py | 15 +- .../tabpy_server/psws/python_service.py | 6 +- .../tabpy_server/state.ini.template | 3 +- .../tabpy_server/static/index.html | 0 .../tabpy_server/static/tableau.png | Bin tabpy/tabpy_tools/__init__.py | 0 {tabpy-tools => tabpy}/tabpy_tools/client.py | 0 .../tabpy_tools/custom_query_object.py | 0 .../tabpy_tools/query_object.py | 0 {tabpy-tools => tabpy}/tabpy_tools/rest.py | 0 .../tabpy_tools/rest_client.py | 0 {tabpy-tools => tabpy}/tabpy_tools/schema.py | 5 +- tabpy/utils/__init__.py | 0 {utils => tabpy/utils}/user_management.py | 8 +- tests/integration/integ_test_base.py | 33 +++- .../resources/deploy_and_evaluate_model.conf | 57 ++++++ .../test_deploy_and_evaluate_model.py | 14 +- .../test_deploy_and_evaluate_model_ssl.py | 3 +- .../test_deploy_model_ssl_off_auth_off.py | 6 +- .../test_deploy_model_ssl_off_auth_on.py | 7 +- .../test_deploy_model_ssl_on_auth_off.py | 5 +- .../test_deploy_model_ssl_on_auth_on.py | 12 +- tests/unit/server_tests/test_config.py | 87 ++++----- .../test_endpoint_file_manager.py | 2 +- .../server_tests/test_endpoint_handler.py | 6 +- .../server_tests/test_endpoints_handler.py | 6 +- .../test_evaluation_plane_handler.py | 6 +- tests/unit/server_tests/test_pwd_file.py | 2 +- .../server_tests/test_service_info_handler.py | 6 +- tests/unit/tools_tests/test_client.py | 2 +- tests/unit/tools_tests/test_rest.py | 31 ++- tests/unit/tools_tests/test_rest_object.py | 2 +- tests/unit/tools_tests/test_schema.py | 41 ++++ utils/set_env.cmd | 2 - utils/set_env.sh | 1 - 99 files changed, 877 insertions(+), 875 deletions(-) delete mode 100755 .gitattributes create mode 100755 .scrutinizer.yml delete mode 100755 .vscode/launch.json create mode 100755 MANIFEST.in delete mode 100755 docs/server-download.md create mode 100755 docs/server-install.md delete mode 100755 docs/server-startup.md create mode 100755 setup.py delete mode 100644 tabpy-server/setup.py delete mode 100644 tabpy-server/tabpy_server/__init__.py delete mode 100644 tabpy-server/tabpy_server/handlers/__init__.py delete mode 100644 tabpy-server/tabpy_server/tabpy.py delete mode 100755 tabpy-tools/setup.py delete mode 100755 tabpy-tools/tabpy_tools/__init__.py rename {models/utils => tabpy}/__init__.py (100%) mode change 100644 => 100755 rename {tabpy-server/tabpy_server/app => tabpy/models}/__init__.py (100%) mode change 100644 => 100755 rename models/setup.py => tabpy/models/deploy_models.py (95%) rename {models => tabpy/models}/scripts/PCA.py (90%) rename {models => tabpy/models}/scripts/SentimentAnalysis.py (79%) rename {tabpy-server/tabpy_server/common => tabpy/models/scripts}/__init__.py (100%) mode change 100644 => 100755 rename {models => tabpy/models}/scripts/tTest.py (85%) rename {tabpy-server/tabpy_server/management => tabpy/models/utils}/__init__.py (100%) rename {models => tabpy/models}/utils/setup_utils.py (79%) create mode 100755 tabpy/tabpy.py rename {tabpy-server/tabpy_server/psws => tabpy/tabpy_server}/__init__.py (100%) rename {tabpy-server => tabpy}/tabpy_server/app/ConfigParameters.py (100%) rename {tabpy-server => tabpy}/tabpy_server/app/SettingsParameters.py (100%) create mode 100644 tabpy/tabpy_server/app/__init__.py rename {tabpy-server => tabpy}/tabpy_server/app/app.py (84%) rename {tabpy-server => tabpy}/tabpy_server/app/util.py (100%) create mode 100644 tabpy/tabpy_server/common/__init__.py rename {tabpy-server => tabpy}/tabpy_server/common/default.conf (79%) rename {tabpy-server => tabpy}/tabpy_server/common/endpoint_file_mgr.py (100%) rename {tabpy-server => tabpy}/tabpy_server/common/messages.py (100%) rename {tabpy-server => tabpy}/tabpy_server/common/util.py (100%) create mode 100644 tabpy/tabpy_server/handlers/__init__.py rename {tabpy-server => tabpy}/tabpy_server/handlers/base_handler.py (99%) rename {tabpy-server => tabpy}/tabpy_server/handlers/endpoint_handler.py (93%) rename {tabpy-server => tabpy}/tabpy_server/handlers/endpoints_handler.py (95%) rename {tabpy-server => tabpy}/tabpy_server/handlers/evaluation_plane_handler.py (97%) rename {tabpy-server => tabpy}/tabpy_server/handlers/main_handler.py (70%) rename {tabpy-server => tabpy}/tabpy_server/handlers/management_handler.py (94%) rename {tabpy-server => tabpy}/tabpy_server/handlers/query_plane_handler.py (97%) rename {tabpy-server => tabpy}/tabpy_server/handlers/service_info_handler.py (86%) rename {tabpy-server => tabpy}/tabpy_server/handlers/status_handler.py (93%) rename {tabpy-server => tabpy}/tabpy_server/handlers/upload_destination_handler.py (79%) rename {tabpy-server => tabpy}/tabpy_server/handlers/util.py (89%) create mode 100644 tabpy/tabpy_server/management/__init__.py rename {tabpy-server => tabpy}/tabpy_server/management/state.py (99%) rename {tabpy-server => tabpy}/tabpy_server/management/util.py (93%) create mode 100644 tabpy/tabpy_server/psws/__init__.py rename {tabpy-server => tabpy}/tabpy_server/psws/callbacks.py (93%) rename {tabpy-server => tabpy}/tabpy_server/psws/python_service.py (98%) rename tabpy-server/tabpy_server/state.ini => tabpy/tabpy_server/state.ini.template (86%) rename {tabpy-server => tabpy}/tabpy_server/static/index.html (100%) rename {tabpy-server => tabpy}/tabpy_server/static/tableau.png (100%) create mode 100755 tabpy/tabpy_tools/__init__.py rename {tabpy-tools => tabpy}/tabpy_tools/client.py (100%) rename {tabpy-tools => tabpy}/tabpy_tools/custom_query_object.py (100%) rename {tabpy-tools => tabpy}/tabpy_tools/query_object.py (100%) rename {tabpy-tools => tabpy}/tabpy_tools/rest.py (100%) rename {tabpy-tools => tabpy}/tabpy_tools/rest_client.py (100%) rename {tabpy-tools => tabpy}/tabpy_tools/schema.py (98%) create mode 100755 tabpy/utils/__init__.py rename {utils => tabpy/utils}/user_management.py (94%) create mode 100755 tests/integration/resources/deploy_and_evaluate_model.conf create mode 100755 tests/unit/tools_tests/test_schema.py delete mode 100755 utils/set_env.cmd delete mode 100755 utils/set_env.sh diff --git a/.coveragerc b/.coveragerc index c82dc401..228ef0f8 100755 --- a/.coveragerc +++ b/.coveragerc @@ -4,4 +4,11 @@ exclude_lines = if __name__ == .__main__.: # Only show one number after decimal point in report. -precision = 1 \ No newline at end of file +precision = 1 + +[run] +omit = + tabpy/models/* + tabpy/tabpy.py + tabpy/utils/* + tests/* diff --git a/.gitattributes b/.gitattributes deleted file mode 100755 index 8f5277c3..00000000 --- a/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -tabpy-server/tabpy_server/_version.py export-subst diff --git a/.gitignore b/.gitignore index a204dcab..621c6072 100644 --- a/.gitignore +++ b/.gitignore @@ -117,12 +117,13 @@ package-lock.json .idea/ # TabPy server artifacts -tabpy-server/install.log -tabpy-server/tabpy_server/query_objects -tabpy-server/tabpy_server/staging +tabpy/tabpy_server/state.ini +tabpy/tabpy_server/query_objects +tabpy/tabpy_server/staging # VS Code *.code-workspace +.vscode # etc setup.bat diff --git a/.scrutinizer.yml b/.scrutinizer.yml new file mode 100755 index 00000000..22572143 --- /dev/null +++ b/.scrutinizer.yml @@ -0,0 +1,24 @@ +build: + environment: + python: 3.6 + nodes: + analysis: + project_setup: + override: + - pip install sklearn pandas numpy textblob nltk scipy + tests: + override: + - py-scrutinizer-run + - + command: pylint-run + use_website_config: true + tests: true +checks: + python: + code_rating: true + duplicate_code: true +filter: + excluded_paths: + - '*/test/*' + dependency_paths: + - 'lib/*' diff --git a/.travis.yml b/.travis.yml index b61f47f2..13cbf053 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,13 +2,12 @@ os: linux language: python python: 3.6 install: - - ./startup.sh --no-startup --print-install-logs - pip install pytest pytest-cov coveralls - npm install -g markdownlint-cli script: - - source utils/set_env.sh - - pytest tests/unit --cov=tabpy-server/tabpy_server --cov=tabpy-tools/tabpy_tools --cov-append - - pytest tests/integration + - pip install -e . + - py.test tests/unit --cov=tabpy --cov-append + - py.test tests/integration --cov=tabpy --cov-append - markdownlint . after_success: - coveralls diff --git a/.vscode/launch.json b/.vscode/launch.json deleted file mode 100755 index 6c044293..00000000 --- a/.vscode/launch.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "name": "Python: General", - "type": "python", - "request": "launch", - "program": "${file}", - "console": "externalTerminal", - "env": {"${PYTHONPATH}": "${PYTHONPATH};${workspaceRoot}/tabpy-server;${workspaceRoot}/tabpy-tools"} - } - ] -} \ No newline at end of file diff --git a/CHANGELOG b/CHANGELOG index 3a8dc478..bd54d1c1 100755 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,7 +1,11 @@ -# TabPy Changelog +# Changelog -This file lists notable changes for TabPy project releases. -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## v0.8 + +### Improvements + +- TabPy is pip package now +- Models are deployed with updated script ## v0.7 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index adb3f35e..43370b0f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,21 +1,26 @@ # TabPy Contributing Guide + + - [Environment Setup](#environment-setup) - [Prerequisites](#prerequisites) - [Cloning TabPy Repository](#cloning-tabpy-repository) -- [Setting Up Environment](#setting-up-environment) -- [Unit Tests](#unit-tests) -- [Integration Tests](#integration-tests) +- [Tests](#tests) + * [Unit Tests](#unit-tests) + * [Integration Tests](#integration-tests) - [Code Coverage](#code-coverage) - [TabPy in Python Virtual Environment](#tabpy-in-python-virtual-environment) - [Documentation Updates](#documentation-updates) - [TabPy with Swagger](#tabpy-with-swagger) - [Code styling](#code-styling) +- [Publishing TabPy Package](#publishing-tabpy-package) + + ## Environment Setup The purpose of this guide is to enable developers of Tabpy to install the project @@ -27,12 +32,18 @@ These are prerequisites for an environment required for a contributor to be able to work on TabPy changes: - Python 3.6.5: - - To see which version of Python you have installed, run ```python --version```. + - To see which version of Python you have installed, run `python --version`. - git - TabPy repo: - - Get the latest TabPy repository with `git clone https://github.com/tableau/TabPy.git` + - Get the latest TabPy repository with + `git clone https://github.com/tableau/TabPy.git`. - Create a new branch for your changes. - When changes are ready push them on github and create merge request. +- PIP packages - install all with + `pip install pytest pycodestyle autopep8 twine --upgrade` command +- Node.js for npm packages - install from . +- NPM packages - install all with + `npm install markdown-toc markdownlint` command. ## Cloning TabPy Repository @@ -46,32 +57,29 @@ be able to work on TabPy changes: cd TabPy ``` -Before making any code changes run environment setup script. -For Windows run this command from the repository root folder: +4. Register TabPy repo as a pip package: -```sh -utils\set_env.cmd -``` + ```sh + pip install -e . + ``` -and for Linux or Mac the next command from the repository root folder: +## Tests + +To run the whole test suite execute the following command: ```sh -source utils/set_env.sh +pytest ``` -## Unit Tests +### Unit Tests -TabPy has test suites for `tabpy-server` and `tabpy-tools` components. -To run the unit tests use `pytest` which you may need to install first -(see [https://docs.pytest.org](https://docs.pytest.org) for details): +Unit tests suite can be executed with the following command: ```sh pytest tests/unit ``` -Check `pytest` documentation for how to run individual tests or set of tests. - -## Integration Tests +### Integration Tests Integration tests can be executed with the next command: @@ -106,13 +114,19 @@ TOC for markdown file is built with [markdown-toc](https://www.npmjs.com/package markdown-toc -i docs/server-startup.md ``` +To check markdown style for all the documentation use `markdownlint`: + +```sh +markdownlint . +``` + These checks will run as part of the build if you submit a pull request. ## TabPy with Swagger You can invoke the TabPy Server API against a running TabPy instance with Swagger. -- Make CORS related changes in TabPy configuration file: update `tabpy-server\state.ini` +- Make CORS related changes in TabPy configuration file: update `tabpy/tabpy-server/state.ini` file in your local repository to have the next settings: ```config @@ -131,32 +145,29 @@ Access-Control-Allow-Methods = GET, OPTIONS, POST ## Code styling -`pycodestyle` is used to check Python code against our style conventions. -You can run install it and run locally for files where modifications were made: +`pycodestyle` is used to check Python code against our style conventions: ```sh -pip install pycodestyle -``` - -And then run it for files where modifications were made, e.g.: - -```sh -pycodestyle tabpy-server/server_tests/test_pwd_file.py +pycodestyle . ``` For reported errors and warnings either fix them manually or auto-format files with `autopep8`. -To install `autopep8` run the next command: +Run the tool for a file. In the example below `-i` +option tells `autopep8` to update the file. Without the option it +outputs formatted code to the console. ```sh -pip install autopep8 +autopep8 -i tabpy-server/server_tests/test_pwd_file.py ``` -Then you can run the tool for a file. In the example below `-i` -option tells `autopep8` to update the file. Without the option it -outputs formatted code to the console. +## Publishing TabPy Package + +Execute the following commands to build and publish a new version of +TabPy package: ```sh -autopep8 -i tabpy-server/server_tests/test_pwd_file.py +python setup.py sdist bdist_wheel +python -m twine upload dist/* ``` diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100755 index 00000000..b119f809 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,7 @@ +include \ + CHANGELOG \ + LICENSE \ + VERSION \ + tabpy/tabpy_server/state.ini \ + tabpy/tabpy_server/static/* \ + tabpy/tabpy_server/common/default.conf diff --git a/README.md b/README.md index 65bbce07..a744f703 100755 --- a/README.md +++ b/README.md @@ -14,13 +14,11 @@ TabPy (the Tableau Python Server) is an external service implementation which ex Tableau's capabilities by allowing users to execute Python scripts and saved functions via Tableau's table calculations. -All documentation is in the [docs](docs) folder. Consider reading it in this -order: +Consider reading TabPy documentation in the following order: * [About TabPy](docs/about.md) -* [TabPy Server Download Instructions](docs/server-download.md) +* [TabPy Installation Instructions](docs/server-install.md) * [TabPy Server Configuration Instructions](docs/server-config.md) -* [TabPy Server Startup Instructions](docs/server-startup.md) * [Running TabPy in Python Virtual Environment](docs/tabpy-virtualenv.md) * [Authoring Python calculations in Tableau](docs/TableauConfiguration.md). * [TabPy Tools](docs/tabpy-tools.md) diff --git a/VERSION b/VERSION index 0e2c9395..ce609caf 100755 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.7 \ No newline at end of file +0.8 \ No newline at end of file diff --git a/docs/FAQ.md b/docs/FAQ.md index f50dd533..77044de7 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -3,6 +3,10 @@ + +- [Startup Issues](#startup-issues) + * [AttributeError: module 'tornado.web' has no attribute 'asynchronous'](#attributeerror-module-tornadoweb-has-no-attribute-asynchronous) + diff --git a/docs/server-config.md b/docs/server-config.md index a57bd363..97d1d354 100755 --- a/docs/server-config.md +++ b/docs/server-config.md @@ -4,11 +4,13 @@ +- [Custom Settings](#custom-settings) + * [Configuration File Content](#configuration-file-content) + * [Configuration File Example](#configuration-file-example) - [Configuring HTTP vs HTTPS](#configuring-http-vs-https) - [Authentication](#authentication) * [Enabling Authentication](#enabling-authentication) * [Password File](#password-file) - * [Setting Up Environment](#setting-up-environment) * [Adding an Account](#adding-an-account) * [Updating an Account](#updating-an-account) * [Deleting an Account](#deleting-an-account) @@ -20,31 +22,135 @@ -Default settings for TabPy may be viewed in the -tabpy_server/common/default.conf file. This file also contains -commented examples of how to set up your TabPy server to only -serve HTTPS traffic and enable authentication. +## Custom Settings -Change settings by: +TabPy starts with set of default settings unless settings are provided via +environment variables or with a config file. -1. Adding environment variables: - - set the environment variable as required by your Operating System. When - creating environment variables, use the same name as is in the config file - as an environment variable. The files startup.sh and startup.cmd in the root - of the install folder have examples of how to set environment variables in - both Linux and Windows respectively. Set any desired environment variables - and then start the application. -2. Modifying default.conf. -3. Specifying your own config file as a command line parameter. - - i.e. Running the command: +Configuration parameters can be updated with: - ```sh - python tabpy.py --config=path\to\my\config - ``` +1. Adding environment variables - set the environment variable as required by + your Operating System. When creating environment variables, use the same + name for your environment variable as specified in the config file. +2. Specifying a parameter in a config file (enviroment variable value overwrites + configuration setting). + +Configuration file with custom settings is specified as a command line parameter: + +```sh +tabpy --config=path/to/my/config/file.conf +``` The default config file is provided to show you the default values but does not need to be present to run TabPy. +### Configuration File Content + +Configuration file consists of settings for TabPy itself and Python logger +settings. You should only set parameters if you need different values than +the defaults. + +TabPy parameters explained below, the logger documentation can be found +at [`logging.config` documentation page](https://docs.python.org/3.6/library/logging.config.html). + +`[TabPy]` parameters: + +- `TABPY_PORT` - port for TabPy to listen on. Default value - `9004`. +- `TABPY_QUERY_OBJECT_PATH` - query objects location. Used with models, see + [TabPy Tools documentation](tabpy-tools.md) for details. Default value - + `/tmp/query_objects`. +- `TABPY_STATE_PATH` - state folder location (absolute path) for Tornado web + server. Default value - `tabpy/tabpy_server` subfolder in TabPy package + folder. +- `TABPY_STATIC_PATH` - absolute path for location of static files (index.html + page) for TabPy instance. Default value - `tabpy/tabpy_server/static` + subfolder in TabPy package folder. +- `TABPY_PWD_FILE` - absolute path to password file. Setting up this parameter + makes TabPy require credentials with HTTP(S) requests. More details about + authentication can be found in [Authentication](#authentication) + section. Default value - not set. +- `TABPY_TRANSFER_PROTOCOL` - transfer protocol. Default value - `http`. If + set to `https` two additional parameters have to be specified: + `TABPY_CERTIFICATE_FILE` and `TABPY_KEY_FILE`. More details for how to + configure TabPy for HTTPS are at [Configuring HTTP vs HTTPS] + (#configuring-http-vs-https) section. +- `TABPY_CERTIFICATE_FILE` - absolute path to the certificate file to run + TabPy with. Only used with `TABPY_TRANSFER_PROTOCOL` set to `https`. + Default value - not set. +- `TABPY_KEY_FILE` - absolute path to private key file to run TabPy with. + Only used with `TABPY_TRANSFER_PROTOCOL` set to `https`. Default value - + not set. +- `TABPY_LOG_DETAILS` - when set to `true` additional call information + (caller IP, URL, client info, etc.) is logged. Default value - `false`. +- `TABPY_EVALUATE_TIMEOUT` - script evaluation timeout in seconds. Default + value - `30`. + +### Configuration File Example + +**Note:** _Always use absolute paths for the configuration paths +settings._ + +```ini +[TabPy] +# TABPY_QUERY_OBJECT_PATH = /tmp/query_objects +# TABPY_PORT = 9004 +# TABPY_STATE_PATH = /tabpy/tabpy_server + +# Where static pages live +# TABPY_STATIC_PATH = /tabpy/tabpy_server/static + +# For how to configure TabPy authentication read +# docs/server-config.md. +# TABPY_PWD_FILE = /path/to/password/file.txt + +# To set up secure TabPy uncomment and modify the following lines. +# Note only PEM-encoded x509 certificates are supported. +# TABPY_TRANSFER_PROTOCOL = https +# TABPY_CERTIFICATE_FILE = /path/to/certificate/file.crt +# TABPY_KEY_FILE = /path/to/key/file.key + +# Log additional request details including caller IP, full URL, client +# end user info if provided. +# TABPY_LOG_DETAILS = true + +# Configure how long a custom script provided to the /evaluate method +# will run before throwing a TimeoutError. +# The value should be a float representing the timeout time in seconds. +#TABPY_EVALUATE_TIMEOUT = 30 + +[loggers] +keys=root + +[handlers] +keys=rootHandler,rotatingFileHandler + +[formatters] +keys=rootFormatter + +[logger_root] +level=DEBUG +handlers=rootHandler,rotatingFileHandler +qualname=root +propagete=0 + +[handler_rootHandler] +class=StreamHandler +level=DEBUG +formatter=rootFormatter +args=(sys.stdout,) + +[handler_rotatingFileHandler] +class=handlers.RotatingFileHandler +level=DEBUG +formatter=rootFormatter +args=('tabpy_log.log', 'a', 1000000, 5) + +[formatter_rootFormatter] +format=%(asctime)s [%(levelname)s] (%(filename)s:%(module)s:%(lineno)d): %(message)s +datefmt=%Y-%m-%d,%H:%M:%S + +``` + ## Configuring HTTP vs HTTPS By default, TabPy serves only HTTP requests. TabPy can be configured to serve @@ -86,41 +192,25 @@ Password file is a text file containing usernames and hashed passwords per line separated by single space. For username only ASCII characters are supported. Usernames are case-insensitive. -Passwords in the password file are hashed with PBKDF2. [See source code -for implementation details](../tabpy-server/tabpy_server/handlers/util.py). +Passwords in the password file are hashed with PBKDF2. **It is highly recommended to restrict access to the password file with hosting OS mechanisms. Ideally the file should only be accessible for reading with the account under which TabPy runs and TabPy admin account.** -There is a `utils/user_management.py` utility to operate with -accounts in the password file. Run `utils/user_management.py -h` to -see how to use it. +There is a `tabpy-user-management` command provided with `tabpy` package to +operate with accounts in the password file. Run `tabpy-user-management -h` +to see how to use it. After making any changes to the password file, TabPy needs to be restarted. -### Setting Up Environment - -Before making any code changes run the environment setup script. For -Windows run this command from the repository root folder: - -```sh -utils\set_env.cmd -``` - -and for Linux or Mac run this command from the repository root folder: - -```sh -source utils/set_env.sh -``` - ### Adding an Account -To add an account run `utils/user_management.py` utility with `add` +To add an account run `tabpy-user-management add` command providing user name, password (optional) and password file: ```sh -python utils/user_management.py add -u -p -f +tabpy-user-management add -u -p -f ``` If the (recommended) `-p` argument is not provided a password for the user name @@ -128,11 +218,11 @@ will be generated and displayed in the command line. ### Updating an Account -To update the password for an account run `utils/user_management.py` utility -with `update` command: +To update the password for an account run `tabpy-user-management update` +command: ```sh -python utils/user_management.py update -u -p -f +tabpy-user-management update -u -p -f ``` If the (recommended) `-p` agrument is not provided a password for the user name diff --git a/docs/server-download.md b/docs/server-download.md deleted file mode 100755 index fe6f45fb..00000000 --- a/docs/server-download.md +++ /dev/null @@ -1,13 +0,0 @@ -# TabPy Server Download - -The TabPy server can be downloaded from -[TabPy github releases page](https://github.com/tableau/TabPy/releases). -Read the description for the releases and then download the appropriate -`zip` or `tar.gz` TabPy archive to your machine. The latest release -of TabPy is strongly recommended unless you have a specific restriction -or package incompatability you need to consider. - -After downloading the archive unpack it to a folder on your machine and -you are ready to start running TabPy from that folder. Read the -[configuration](server-config.md) and [starting up instructions](server-startup.md) -for more details. diff --git a/docs/server-install.md b/docs/server-install.md new file mode 100755 index 00000000..e91aa4dc --- /dev/null +++ b/docs/server-install.md @@ -0,0 +1,49 @@ +# TabPy Installation Instructions + +These instructions explain how to install and start up TabPy Server. + + + + + +- [TabPy Installation](#tabpy-installation) +- [Starting TabPy](#starting-tabpy) + + + + + +## TabPy Installation + +To install TabPy on to an environment `pip` needs to be installed and +updated first: + +```sh +python -m pip install --upgrade pip +``` + +Now TabPy can be install as a package: + +```sh +pip install tabpy +``` + +## Starting TabPy + +To start TabPy with default setting run the following command: + +```sh +tabpy +``` + +To run TabPy with custom settings create config file with parameters +explained in [TabPy Server Configuration Instructions](server-config.md) +and specify it in command line: + +```sh +tabpy --config=path/to/my/config/file.conf +``` + +It is highly recommended to use Python virtual enviroment for running TabPy. +Check the [Running TabPy in Python Virtual Environment](tabpy-virtualenv.md) page +for more details. diff --git a/docs/server-startup.md b/docs/server-startup.md deleted file mode 100755 index 27e8ec57..00000000 --- a/docs/server-startup.md +++ /dev/null @@ -1,127 +0,0 @@ -# Setup and Startup TabPy Server - -These instructions explain how to start up TabPy Server. - - - - -- [Prerequisites](#prerequisites) -- [Windows](#windows) - * [Command Line Arguments](#command-line-arguments) -- [Mac](#mac) - * [Command Line Arguments](#command-line-arguments-1) -- [Linux](#linux) - * [Command Line Arguments](#command-line-arguments-2) - - - - -## Prerequisites - -To start up TabPy Server from an environment the following prerequisites are required: - -- Python 3.6.5 -- setuptools (Python module, can be installed from PyPi) - -First, select a TabPy version and download its source code from the -[releases page](https://github.com/tableau/TabPy/releases). To start up -a TabPy server instance, follow the instructions for your OS (found below). - -Instructions on how to configure your TabPy server instance can be found in the -[TabPy Server Configuration Instructions](server-config.md) - -It is highly recommended to use Python virtual enviroment for running TabPy. -Check the [Running TabPy in Python Virtual Environment](tabpy-virtualenv.md) page -for more details. -If you are installing a newer version of TabPy in the same environment as a -previous install, delete the previous TabPy version folder in your Python directory. - -## Windows - -1. Open a command prompt. -2. Navigate to the folder in which you downloaded your source code. - - This folder should contain the file: ```startup.cmd``` -3. Run the following command from the command prompt: - - ```batch - startup.cmd - ``` - -### Command Line Arguments for Windows - -To specify the *config file* with which to configure your server instance, pass -it in as a command line argument as follows: - -```batch -startup.cmd myconfig.conf -``` - -Replace `myconfig.conf` with the path to your config file relative to -`%TABPY_ROOT%\`. - -For example, in this case your config file would be located at -`%TABPY_ROOT%\myconfig.conf`. - -## Mac - -1. Open a terminal. -2. Navigate to the folder in which you downloaded your source code. - - This folder should contain the file: ```startup.sh``` -3. Run the following command from the terminal: - - ```bash - ./startup.sh - ``` - -### Command Line Arguments for Mac - -- To specify the *config file* with which to configure your server instance, - set the ```--config=*``` or ```-c=*``` command line argument as follows: - - ```bash - ./startup.sh --config=myconfig.conf - ``` - - or - - ```bash - ./startup.sh -c=myconfig.conf - ``` - - Replace ```myconfig.conf``` with the path to your config file relative to - ```$TABPY_ROOT/```. - - For example, in this case your config file would be located at - ```$TABPY_ROOT/myconfig.conf```. - -## Linux - -1. Open a terminal. -2. Navigate to the folder in which you downloaded your source code. - - This folder should contain the file: ```startup.sh``` -3. Run the following command from the terminal: - - ```bash - ./startup.sh - ``` - -### Command Line Arguments for Linux - -- To specify the *config file* with which to configure your server instance, - set the ```--config=*``` or ```-c=*``` command line argument as follows: - - ```bash - ./startup.sh --config=myconfig.conf - ``` - - or - - ```bash - ./startup.sh -c=myconfig.conf - ``` - - Replace ```myconfig.conf``` with the path to your config file relative to - ```$TABPY_ROOT/```. - - For example, in this case your config file would be located at - ```$TABPY_ROOT/myconfig.conf```. diff --git a/docs/tabpy-tools.md b/docs/tabpy-tools.md index d46839f0..07ca73c4 100755 --- a/docs/tabpy-tools.md +++ b/docs/tabpy-tools.md @@ -3,18 +3,25 @@ TabPy tools is the Python package of tools for managing the published Python functions on TabPy server. + + - [Connecting to TabPy](#connecting-to-tabpy) - [Authentication](#authentication) - [Deploying a Function](#deploying-a-function) - [Predeployed Functions](#predeployed-functions) + * [Principal Component Analysis (PCA)](#principal-component-analysis-pca) + * [Sentiment Analysis](#sentiment-analysis) + * [T-Test](#t-test) - [Providing Schema Metadata](#providing-schema-metadata) - [Querying an Endpoint](#querying-an-endpoint) - [Evaluating Arbitrary Python Scripts](#evaluating-arbitrary-python-scripts) + + ## Connecting to TabPy The tools library uses the notion of connecting to a service to avoid having @@ -39,9 +46,7 @@ has to specify the credentials to use during model deployment with the `set_credentials` call for a client: ```python - client.set_credentials('username', 'P@ssw0rd') - ``` Credentials only need to be set once for all further client operations. @@ -57,13 +62,11 @@ TabPy, see [TabPy Server Configuration Instructions](server-config.md). A persisted endpoint is backed by a Python method. For example: ```python - def add(x,y): import numpy as np return np.add(x, y).tolist() client.deploy('add', add, 'Adds two numbers x and y') - ``` The next example is more complex, using scikit-learn's clustering API: @@ -99,9 +102,7 @@ You can re-deploy a function (for example, after you modified its code) by setti the `override` parameter to `True`: ```python - client.deploy('add', add, 'Adds two numbers x and y', override=True) - ``` Each re-deployment of an endpoint will increment its version number, which is also @@ -135,7 +136,6 @@ executed. In order to get the best performance, we recommended following the methodology outlined in this example. ```python - def LoanDefaultClassifier(Loan_Amount, Loan_Tenure, Monthly_Income, Age): import pandas as pd data=pd.concat([Loan_Amount,Loan_Tenure,Monthly_Income,Age],axis=1) @@ -144,12 +144,12 @@ def LoanDefaultClassifier(Loan_Amount, Loan_Tenure, Monthly_Income, Age): client.deploy('WillItDefault', LoanDefaultClassifier, 'Returns whether a loan application is likely to default.') - ``` You can find a detailed working example with a downloadable sample Tableau workbook and an accompanying Jupyter workbook that walks through model fitting, evaluation -and publishing steps on [our blog](https://www.tableau.com/about/blog/2017/1/building-advanced-analytics-applications-tabpy-64916). +and publishing steps on +[our blog](https://www.tableau.com/about/blog/2017/1/building-advanced-analytics-applications-tabpy-64916). The endpoints that are no longer needed can be removed the following way: @@ -161,34 +161,41 @@ client.remove('WillItDefault') ## Predeployed Functions -To setup models, download the latest version of TabPy and follow the [instructions](server-download.md) -to install and start up your server. Once your server is running, navigate to the -models directory and run setup.py. If your TabPy server is running on the default -config (default.conf), you do not need to specify a config file when launching the -script. If your server is running using a custom config you can specify the config -in the command line like so: +### Deploying Models Shipped With TabPy + +To deploy models shipped with TabPy follow the +[TabPy Installation Instructions](server-install.md) and then +[TabPy Server Configuration Instructions](server-config.md). +Once your server is running execute the following command: ```sh +tabpy-deploy-models +``` -python setup.py custom.conf +If your server is running using a custom config specify the config +in the command line: +```sh +tabpy-deploy-models custom.conf ``` -The setup file will install all of the necessary dependencies `(eg. sklearn, -nltk, textblob, pandas, & numpy)` and deploy all of the prebuilt models -located in `./models/scripts`. For every model that is successfully deployed -a message will be printed to the console: +The command will install all of the necessary dependencies (e.g. `sklearn`, +`nltk`, `textblob`, `pandas`, `numpy`) and deploy all of the prebuilt models. +For every successfully deployed model a message will be printed to the console: ```sh "Successfully deployed PCA" ``` -If you would like to deploy additional models using the deploy script, you can -copy any python file to the `./models/scripts` directory and modify setup.py to -include all necessary packages when installing dependencies, or alternatively install +Use code in [`tabpy/models/scripts`](../tabpy/models/scripts) +as an example of how to create a model and +[`tabpy/models/deploy_models.py`](../tabpy/models/deploy_models.py) +as an example for how to deploy a model. For deployment script include all +necessary packages when installing dependencies or alternatively install all the required dependencies manually. -You can deploy models individually by navigating to models/scripts/ and running +You can deploy models individually by navigating to +[`tabpy/models/scripts`](../tabpy/models/scripts) and running each file in isolation like so: ```sh diff --git a/docs/tabpy-virtualenv.md b/docs/tabpy-virtualenv.md index 3ce31ce0..12a39497 100755 --- a/docs/tabpy-virtualenv.md +++ b/docs/tabpy-virtualenv.md @@ -2,10 +2,7 @@ -To run TabPy in Python virtual environment follow steps -below. - -## Windows Specific Steps +To run TabPy in Python virtual environment follow the steps: 1. Install `virtualenv` package: @@ -13,54 +10,30 @@ below. pip install virtualenv ``` -2. Create virtual environment: - - ```sh - virtualenv - ``` - -3. Activate the environment: - - ```sh - \Scripts\activate - ``` - -4. Run TabPy: +2. Create virtual environment (replace `my-tabpy-env` with + your virtual environment name): ```sh - startup.cmd + virtualenv my-tabpy-env ``` -5. To deactivate virtual environment run: +3. Activate the environment. + 1. For Windows run - ```sh - deactivate - ``` + ```sh + my-tabpy-env\Scripts\activate + ``` -## Linux and Mac Specific Steps + 2. For Linux and Mac run -1. Install `virtualenv` package: - - ```sh - pip install virtualenv - ``` - -2. Create virtual environment: - - ```sh - virtualenv - ``` - -3. Activate the environment: - - ```sh - /bin/activate - ``` + ```sh + my-tabpy-env/bin/activate + ``` 4. Run TabPy: ```sh - ./startup.sh + tabpy ``` 5. To deactivate virtual environment run: diff --git a/setup.py b/setup.py new file mode 100755 index 00000000..8676a7d4 --- /dev/null +++ b/setup.py @@ -0,0 +1,105 @@ +'''Web server Tableau uses to run Python scripts. + +TabPy (the Tableau Python Server) is an external service implementation +which expands Tableau's capabilities by allowing users to execute Python +scripts and saved functions via Tableau's table calculations. +''' + +import os +from setuptools import setup, find_packages + + +DOCLINES = (__doc__ or '').split('\n') + + +def setup_package(): + def read(fname): + return open(os.path.join(os.path.dirname(__file__), fname)).read() + + setup( + name='tabpy', + version=read('VERSION'), + description=DOCLINES[0], + long_description='\n'.join(DOCLINES[1:]) + '\n' + read('CHANGELOG'), + long_description_content_type='text/markdown', + url='https://github.com/tableau/TabPy', + author='Tableau', + author_email='github@tableau.com', + maintainer='Tableau', + maintainer_email='github@tableau.com', + download_url='https://pypi.org/project/tabpy', + project_urls={ + "Bug Tracker": "https://github.com/tableau/TabPy/issues", + "Documentation": "https://tableau.github.io/TabPy/", + "Source Code": "https://github.com/tableau/TabPy", + }, + classifiers=[ + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'Intended Audience :: Science/Research', + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python :: 3.6', + 'Topic :: Scientific/Engineering', + 'Topic :: Scientific/Engineering :: Information Analysis', + 'Operating System :: Microsoft :: Windows', + 'Operating System :: POSIX', + 'Operating System :: Unix', + 'Operating System :: MacOS' + ], + platforms=['Windows', 'Linux', 'Mac OS-X', 'Unix'], + keywords=['tabpy tableau'], + packages=find_packages( + exclude=['docs', 'misc', 'tests']), + package_data={ + 'tabpy': [ + 'VERSION', + 'tabpy_server/state.ini', + 'tabpy_server/static', + 'tabpy_server/common/default.conf' + ] + }, + python_requires='>=3.6', + license='MIT', + # Note: many of these required packages are included in base python + # but are listed here because different linux distros use custom + # python installations. And users can remove packages at any point + install_requires=[ + 'backports_abc', + 'cloudpickle', + 'configparser', + 'decorator', + 'future', + 'genson', + 'jsonschema', + 'pyopenssl', + 'python-dateutil', + 'requests', + 'singledispatch', + 'six', + 'tornado', + 'urllib3<1.25,>=1.21.1' + ], + entry_points={ + 'console_scripts': [ + 'tabpy=tabpy.tabpy:main', + 'tabpy-deploy-models=tabpy.models.deploy_models:main', + 'tabpy-user-management=tabpy.utils.user_management:main' + ], + }, + setup_requires=['pytest-runner'], + tests_require=[ + 'mock', + 'nltk', + 'numpy', + 'pandas', + 'pytest', + 'scipy', + 'sklearn', + 'textblob' + ], + test_suite='pytest' + ) + + +if __name__ == '__main__': + setup_package() diff --git a/startup.cmd b/startup.cmd index a25caa68..48ddf7e7 100755 --- a/startup.cmd +++ b/startup.cmd @@ -1,87 +1,3 @@ @ECHO off -SETLOCAL - - -REM Set environment variables. -SET TABPY_ROOT="%CD%" -SET INSTALL_LOG=%TABPY_ROOT%\tabpy-server\install.log -SET SAVE_PYTHONPATH=%PYTHONPATH% -SET MIN_PY_VER=3.6 -SET DESIRED_PY_VER=3.6.5 - - -ECHO Checking for presence of Python in the system path variable. -SET PYTHON_ERROR=Fatal Error : TabPy startup failed. Check that Python 3.6.5 or higher is installed and is in the system PATH environment variable. -python --version -IF %ERRORLEVEL% NEQ 0 ( - ECHO %PYTHON_ERROR% - SET RET=1 - GOTO:END -) ELSE ( - FOR /F "TOKENS=2" %%a IN ('python --version 2^>^&1') DO ( - IF %%a LSS %MIN_PY_VER% ( - ECHO %PYTHON_ERROR% - SET RET=1 - GOTO:END - ) ELSE IF %%a LSS %DESIRED_PY_VER% ( - ECHO Warning : Python %%a% is not supported. Please upgrade Python to 3.6.5 or higher. - SET RET=1 - ) - ) -) - -REM Install requirements using Python setup tools. -ECHO Installing any missing dependencies... - -CD %TABPY_ROOT%\tabpy-server -ECHO Installing tabpy-server dependencies...>%INSTALL_LOG% -python setup.py install>>%INSTALL_LOG% 2>&1 - -CD %TABPY_ROOT%\tabpy-tools -ECHO: >> %INSTALL_LOG% -ECHO Installing tabpy-tools dependencies...>>%INSTALL_LOG% -python setup.py install>>%INSTALL_LOG% 2>&1 - -CD %TABPY_ROOT% -SET INSTALL_LOG_MESSAGE= Check %INSTALL_LOG% for details. -IF %ERRORLEVEL% NEQ 0 ( - CD %TABPY_ROOT% - ECHO failed - ECHO %INSTALL_LOG_MESSAGE% - SET RET=1 - GOTO:END -) ELSE ( - ECHO success - ECHO %INSTALL_LOG_MESSAGE% -) - - -REM Parse optional CLI arguments: config file -ECHO Parsing parameters... -SET PYTHONPATH=.\tabpy-server;.\tabpy-tools;%PYTHONPATH% -SET STARTUP_CMD=python tabpy-server\tabpy_server\tabpy.py -IF [%1] NEQ [] ( - ECHO Using config file at %1 - SET STARTUP_CMD=%STARTUP_CMD% --config=%1 -) - - -ECHO Starting TabPy server... -ECHO; -%STARTUP_CMD% -IF %ERRORLEVEL% NEQ 0 ( - ECHO Failed to start TabPy server. - SET RET=1 - GOTO:END -) - - -SET RET=%ERRORLEVEL% -GOTO:END - - -:END - SET PYTHONPATH=%SAVE_PYTHONPATH% - CD %TABPY_ROOT% - EXIT /B %RET% - ENDLOCAL +ECHO TabPy is a PIP package now, install it with "pip install tabpy". +ECHO For more information read https://tableau.github.io/TabPy/. diff --git a/startup.sh b/startup.sh index 8724a22f..1057d0b4 100755 --- a/startup.sh +++ b/startup.sh @@ -1,115 +1,3 @@ #!/bin/bash - -min_py_ver=3.6 -desired_py_ver=3.6.5 - -function check_status() { - if [ $? -ne 0 ]; then - echo $1 - exit 1 - fi -} - -function check_python_version() { - python3 --version - check_status $1 - - py_ver=($(python3 --version 2>&1) \| tr ' ' ' ') - if [ "${py_ver[1]}" \< "$min_py_ver" ]; then - echo Fatal Error : $1 - exit 1 - elif [ "${py_ver[1]}" \< "$desired_py_ver" ]; then - echo Warning : Python ${py_ver[1]} is not supported. Please upgrade Python to 3.6.5 or higher. - fi -} - -function install_dependencies() { - # $1 = tabpy_server | tabpy_tools - # $2 = true if install logs are printed to the console, - # false if they are printed to a log file - # $3 = install log file path - if [ "$2" = true ]; then - echo -e "\nInstalling ${1} dependencies..." - python3 setup.py install - elif [ "$2" = false ]; then - echo -e "\nInstalling ${1} dependencies..." >> "${3}" - python3 setup.py install >> "${3}" 2>&1 - else - echo Invalid startup environment. - exit 1 - fi - check_status "Cannot install dependencies." -} - -# Check for Python in PATH -echo Checking for presence of Python in the system path variable. -check_python_version "TabPy startup failed. Check that Python 3.6.5 or higher is installed and is in the system PATH environment variable." - -# Setting local variables -echo Setting TABPY_ROOT to current working directory. -TABPY_ROOT="$PWD" -INSTALL_LOG="${TABPY_ROOT}/tabpy-server/install.log" -echo "" > "${INSTALL_LOG}" -PRINT_INSTALL_LOGS=false - -# Parse CLI parameters -for i in "$@" -do - case $i in - -c=*|--config=*) - CONFIG="${i#*=}" - shift - ;; - --no-startup) - NO_STARTUP=true - shift - ;; - --print-install-logs) - PRINT_INSTALL_LOGS=true - shift - ;; - *) - echo Invalid option: $i - esac -done - -# Check for dependencies, install them if they're not present. -echo Installing TabPy-server requirements. -if [ "$PRINT_INSTALL_LOGS" = false ]; then - echo Read the logs at ${INSTALL_LOG} -fi - -cd "${TABPY_ROOT}/tabpy-server" -install_dependencies "tabpy-server" ${PRINT_INSTALL_LOGS} ${INSTALL_LOG} - -cd "${TABPY_ROOT}/tabpy-tools" -install_dependencies "tabpy-tools" ${PRINT_INSTALL_LOGS} ${INSTALL_LOG} - -cd "${TABPY_ROOT}" -check_status - -if [ ! -z ${CONFIG} ]; then - echo Using the config file at ${TABPY_ROOT}/tabpy-server/$CONFIG. -fi - -# Exit if in a test environent -if [ ! -z ${NO_STARTUP} ]; then - echo Skipping server startup. Exiting successfully. - exit 0 -fi - -# Start TabPy server -echo -echo Starting TabPy server... -SAVE_PYTHONPATH=$PYTHONPATH -export PYTHONPATH="${TABPY_ROOT}/tabpy-server:${TABPY_ROOT}/tabpy-tools:$PYTHONPATH" -if [ -z $CONFIG ]; then - echo Using default parameters. - python3 tabpy-server/tabpy_server/tabpy.py -else - python3 tabpy-server/tabpy_server/tabpy.py --config=$CONFIG -fi - -export PYTHONPATH=$SAVE_PYTHONPATH -check_status -exit 0 +echo TabPy is a PIP package now, install it with "pip install tabpy". +echo For more information read https://tableau.github.io/TabPy/. diff --git a/tabpy-server/setup.py b/tabpy-server/setup.py deleted file mode 100644 index 8da8a3c0..00000000 --- a/tabpy-server/setup.py +++ /dev/null @@ -1,51 +0,0 @@ -try: - from setuptools import setup -except ImportError as err: - print("Missing Python module requirement: setuptools.") - raise err - -from tabpy_server import __version__ - -setup( - name='tabpy-server', - version=__version__, - description='Web server Tableau uses to run Python scripts.', - url='https://github.com/tableau/TabPy', - author='Tableau', - author_email='github@tableau.com', - classifiers=[ - 'Development Status :: 4 - Beta', - 'Intended Audience :: Developers', - 'Programming Language :: Python :: 3.6', - "Topic :: Scientific/Engineering", - "Topic :: Scientific/Engineering :: Information Analysis", - ], - packages=['tabpy_server', - 'tabpy_server.common', - 'tabpy_server.management', - 'tabpy_server.psws', - 'tabpy_server.static'], - package_data={'tabpy_server.static': ['*.*'], - 'tabpy_server': ['startup.*', 'state.ini']}, - license='MIT', - # Note: many of these required packages are included in base python - # but are listed here because different linux distros use custom - # python installations. And users can remove packages at any point - install_requires=[ - 'backports_abc', - 'cloudpickle', - 'configparser', - 'decorator', - 'future', - 'genson', - 'mock', - 'numpy', - 'pyopenssl', - 'python-dateutil', - 'requests', - 'singledispatch', - 'six', - 'tornado', - 'urllib3<1.25,>=1.21.1' - ] -) diff --git a/tabpy-server/tabpy_server/__init__.py b/tabpy-server/tabpy_server/__init__.py deleted file mode 100644 index 7b1e011e..00000000 --- a/tabpy-server/tabpy_server/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -from pathlib import Path - - -def read_version(): - f = None - for path in ['VERSION', '../VERSION', '../../VERSION']: - if Path(path).exists(): - f = path - break - - if f is not None: - with open(f) as h: - return h.read().strip() - else: - return 'dev' - - -__version__ = read_version() diff --git a/tabpy-server/tabpy_server/handlers/__init__.py b/tabpy-server/tabpy_server/handlers/__init__.py deleted file mode 100644 index 7dc480b4..00000000 --- a/tabpy-server/tabpy_server/handlers/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -from tabpy_server.handlers.base_handler import BaseHandler -from tabpy_server.handlers.main_handler import MainHandler -from tabpy_server.handlers.management_handler import ManagementHandler - -from tabpy_server.handlers.endpoint_handler import EndpointHandler -from tabpy_server.handlers.endpoints_handler import EndpointsHandler -from tabpy_server.handlers.evaluation_plane_handler\ - import EvaluationPlaneHandler -from tabpy_server.handlers.query_plane_handler import QueryPlaneHandler -from tabpy_server.handlers.service_info_handler import ServiceInfoHandler -from tabpy_server.handlers.status_handler import StatusHandler -from tabpy_server.handlers.upload_destination_handler\ - import UploadDestinationHandler diff --git a/tabpy-server/tabpy_server/tabpy.py b/tabpy-server/tabpy_server/tabpy.py deleted file mode 100644 index 7df99751..00000000 --- a/tabpy-server/tabpy_server/tabpy.py +++ /dev/null @@ -1,10 +0,0 @@ -from tabpy_server.app.app import TabPyApp - - -def main(): - app = TabPyApp() - app.run() - - -if __name__ == '__main__': - main() diff --git a/tabpy-tools/setup.py b/tabpy-tools/setup.py deleted file mode 100755 index 63a9ae64..00000000 --- a/tabpy-tools/setup.py +++ /dev/null @@ -1,30 +0,0 @@ -try: - from setuptools import setup -except ImportError: - from distutils.core import setup - -from tabpy_tools import __version__ - -setup( - name='tabpy-tools', - version=__version__, - description='Python library of tools to manage TabPy Server.', - url='https://github.com/tableau/TabPy', - author='Tableau', - author_email='github@tableau.com', - # see classifiers at https://pypi.org/pypi?:action=list_classifiers - classifiers=[ - 'Development Status :: 4 - Beta', - 'Intended Audience :: Developers', - 'Programming Language :: Python :: 3.5', - "Topic :: Scientific/Engineering", - "Topic :: Scientific/Engineering :: Information Analysis", - ], - packages=['tabpy_tools'], - license='MIT', - install_requires=[ - 'cloudpickle', - 'requests', - 'genson' - ] -) diff --git a/tabpy-tools/tabpy_tools/__init__.py b/tabpy-tools/tabpy_tools/__init__.py deleted file mode 100755 index 27ad0f0a..00000000 --- a/tabpy-tools/tabpy_tools/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -""" -TabPy client is a Python client to interact with a Tornado-Python-Connector -server process. -""" - -from pathlib import Path - - -def read_version(): - f = None - for path in ['VERSION', '../VERSION', '../../VERSION']: - if Path(path).exists(): - f = path - break - - if f is not None: - with open(f) as h: - return h.read().strip() - else: - return 'dev' - - -__version__ = read_version() diff --git a/models/utils/__init__.py b/tabpy/__init__.py old mode 100644 new mode 100755 similarity index 100% rename from models/utils/__init__.py rename to tabpy/__init__.py diff --git a/tabpy-server/tabpy_server/app/__init__.py b/tabpy/models/__init__.py old mode 100644 new mode 100755 similarity index 100% rename from tabpy-server/tabpy_server/app/__init__.py rename to tabpy/models/__init__.py diff --git a/models/setup.py b/tabpy/models/deploy_models.py similarity index 95% rename from models/setup.py rename to tabpy/models/deploy_models.py index aa337758..30dda263 100644 --- a/models/setup.py +++ b/tabpy/models/deploy_models.py @@ -5,7 +5,7 @@ import runpy import subprocess from pathlib import Path -from utils import setup_utils +from tabpy.models.utils import setup_utils # pip 10.0 introduced a breaking change that moves the location of main try: @@ -22,7 +22,7 @@ def install_dependencies(packages): pip._internal.main(pip_arg) -if __name__ == '__main__': +def main(): install_dependencies(['sklearn', 'pandas', 'numpy', 'textblob', 'nltk', 'scipy']) print('==================================================================') @@ -48,3 +48,7 @@ def install_dependencies(packages): for filename in os.listdir(directory): subprocess.run([py, f'{directory}/{filename}', config_file_path] + auth_args) + + +if __name__ == '__main__': + main() diff --git a/models/scripts/PCA.py b/tabpy/models/scripts/PCA.py similarity index 90% rename from models/scripts/PCA.py rename to tabpy/models/scripts/PCA.py index 9c2d68b2..2a2d6b20 100644 --- a/models/scripts/PCA.py +++ b/tabpy/models/scripts/PCA.py @@ -6,8 +6,7 @@ from sklearn.preprocessing import OneHotEncoder import sys from pathlib import Path -sys.path.append(str(Path(__file__).resolve().parent.parent.parent / 'models')) -from utils import setup_utils +from tabpy.models.utils import setup_utils def PCA(component, _arg1, _arg2, *_argN): @@ -59,6 +58,7 @@ def PCA(component, _arg1, _arg2, *_argN): if __name__ == '__main__': - setup_utils.main('PCA', - PCA, - 'Returns the specified principal component') + setup_utils.deploy_model( + 'PCA', + PCA, + 'Returns the specified principal component') diff --git a/models/scripts/SentimentAnalysis.py b/tabpy/models/scripts/SentimentAnalysis.py similarity index 79% rename from models/scripts/SentimentAnalysis.py rename to tabpy/models/scripts/SentimentAnalysis.py index 0b3c3ab6..d3b97e7a 100644 --- a/models/scripts/SentimentAnalysis.py +++ b/tabpy/models/scripts/SentimentAnalysis.py @@ -3,8 +3,12 @@ from nltk.sentiment.vader import SentimentIntensityAnalyzer import sys from pathlib import Path -sys.path.append(str(Path(__file__).resolve().parent.parent.parent / 'models')) -from utils import setup_utils +from tabpy.models.utils import setup_utils + + +import ssl +_ctx = ssl._create_unverified_context +ssl._create_default_https_context = _ctx nltk.download('vader_lexicon') @@ -42,7 +46,8 @@ def SentimentAnalysis(_arg1, library='nltk'): if __name__ == '__main__': - setup_utils.main('Sentiment Analysis', - SentimentAnalysis, - 'Returns a sentiment score between -1 and 1 for ' - 'a given string') + setup_utils.deploy_model( + 'Sentiment Analysis', + SentimentAnalysis, + 'Returns a sentiment score between -1 and 1 for ' + 'a given string') diff --git a/tabpy-server/tabpy_server/common/__init__.py b/tabpy/models/scripts/__init__.py old mode 100644 new mode 100755 similarity index 100% rename from tabpy-server/tabpy_server/common/__init__.py rename to tabpy/models/scripts/__init__.py diff --git a/models/scripts/tTest.py b/tabpy/models/scripts/tTest.py similarity index 85% rename from models/scripts/tTest.py rename to tabpy/models/scripts/tTest.py index d7082698..9bbc0823 100644 --- a/models/scripts/tTest.py +++ b/tabpy/models/scripts/tTest.py @@ -1,8 +1,7 @@ from scipy import stats import sys from pathlib import Path -sys.path.append(str(Path(__file__).resolve().parent.parent.parent / 'models')) -from utils import setup_utils +from tabpy.models.utils import setup_utils def ttest(_arg1, _arg2): @@ -39,6 +38,7 @@ def ttest(_arg1, _arg2): if __name__ == '__main__': - setup_utils.main('ttest', - ttest, - 'Returns the p-value form a t-test') + setup_utils.deploy_model( + 'ttest', + ttest, + 'Returns the p-value form a t-test') diff --git a/tabpy-server/tabpy_server/management/__init__.py b/tabpy/models/utils/__init__.py similarity index 100% rename from tabpy-server/tabpy_server/management/__init__.py rename to tabpy/models/utils/__init__.py diff --git a/models/utils/setup_utils.py b/tabpy/models/utils/setup_utils.py similarity index 79% rename from models/utils/setup_utils.py rename to tabpy/models/utils/setup_utils.py index a5550257..65468153 100644 --- a/models/utils/setup_utils.py +++ b/tabpy/models/utils/setup_utils.py @@ -1,13 +1,16 @@ import configparser -from pathlib import Path import getpass +import os +from pathlib import Path import sys -from tabpy_tools.client import Client +from tabpy.tabpy_tools.client import Client def get_default_config_file_path(): - config_file_path = str(Path(__file__).resolve().parent.parent.parent - / 'tabpy-server/tabpy_server/common/default.conf') + import tabpy + pkg_path = os.path.dirname(tabpy.__file__) + config_file_path = os.path.join( + pkg_path, 'tabpy_server', 'common', 'default.conf') return config_file_path @@ -15,7 +18,11 @@ def parse_config(config_file_path): config = configparser.ConfigParser() config.read(config_file_path) tabpy_config = config['TabPy'] - port = tabpy_config['TABPY_PORT'] + + port = 9004 + if 'TABPY_PORT' in tabpy_config: + port = tabpy_config['TABPY_PORT'] + auth_on = 'TABPY_PWD_FILE' in tabpy_config ssl_on = 'TABPY_TRANSFER_PROTOCOL' in tabpy_config and \ 'TABPY_CERTIFICATE_FILE' in tabpy_config and \ @@ -34,8 +41,8 @@ def get_creds(): return [user, passwd] -def main(funcName, func, funcDescription): - # running from setup.py +def deploy_model(funcName, func, funcDescription): + # running from deploy_models.py if len(sys.argv) > 1: config_file_path = sys.argv[1] else: diff --git a/tabpy/tabpy.py b/tabpy/tabpy.py new file mode 100755 index 00000000..b3e095b7 --- /dev/null +++ b/tabpy/tabpy.py @@ -0,0 +1,36 @@ +''' +TabPy application. +This file main() function is an entry point for +'tabpy' command. +''' + +import os +from pathlib import Path + + +def read_version(): + ver = 'unknonw' + + import tabpy + pkg_path = os.path.dirname(tabpy.__file__) + ver_file_path = os.path.join(pkg_path, os.path.pardir, 'VERSION') + if Path(ver_file_path).exists(): + with open(ver_file_path) as f: + ver = f.read().strip() + else: + ver = f'Version Unknown, (file {ver_file_path} not found)' + + return ver + + +__version__ = read_version() + + +def main(): + from tabpy.tabpy_server.app.app import TabPyApp + app = TabPyApp() + app.run() + + +if __name__ == '__main__': + main() diff --git a/tabpy-server/tabpy_server/psws/__init__.py b/tabpy/tabpy_server/__init__.py similarity index 100% rename from tabpy-server/tabpy_server/psws/__init__.py rename to tabpy/tabpy_server/__init__.py diff --git a/tabpy-server/tabpy_server/app/ConfigParameters.py b/tabpy/tabpy_server/app/ConfigParameters.py similarity index 100% rename from tabpy-server/tabpy_server/app/ConfigParameters.py rename to tabpy/tabpy_server/app/ConfigParameters.py diff --git a/tabpy-server/tabpy_server/app/SettingsParameters.py b/tabpy/tabpy_server/app/SettingsParameters.py similarity index 100% rename from tabpy-server/tabpy_server/app/SettingsParameters.py rename to tabpy/tabpy_server/app/SettingsParameters.py diff --git a/tabpy/tabpy_server/app/__init__.py b/tabpy/tabpy_server/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tabpy-server/tabpy_server/app/app.py b/tabpy/tabpy_server/app/app.py similarity index 84% rename from tabpy-server/tabpy_server/app/app.py rename to tabpy/tabpy_server/app/app.py index 3e91d1e5..da9f3cc8 100644 --- a/tabpy-server/tabpy_server/app/app.py +++ b/tabpy/tabpy_server/app/app.py @@ -5,20 +5,23 @@ from logging import config import multiprocessing import os -import tabpy_server -from tabpy_server import __version__ -from tabpy_server.app.ConfigParameters import ConfigParameters -from tabpy_server.app.SettingsParameters import SettingsParameters -from tabpy_server.app.util import parse_pwd_file -from tabpy_server.management.state import TabPyState -from tabpy_server.management.util import _get_state_from_file -from tabpy_server.psws.callbacks import (init_model_evaluator, init_ps_server) -from tabpy_server.psws.python_service import (PythonService, - PythonServiceHandler) -from tabpy_server.handlers import (EndpointHandler, EndpointsHandler, - EvaluationPlaneHandler, QueryPlaneHandler, - ServiceInfoHandler, StatusHandler, - UploadDestinationHandler) +import shutil +import tabpy.tabpy_server +from tabpy.tabpy import __version__ +from tabpy.tabpy_server.app.ConfigParameters import ConfigParameters +from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters +from tabpy.tabpy_server.app.util import parse_pwd_file +from tabpy.tabpy_server.management.state import TabPyState +from tabpy.tabpy_server.management.util import _get_state_from_file +from tabpy.tabpy_server.psws.callbacks\ + import (init_model_evaluator, init_ps_server) +from tabpy.tabpy_server.psws.python_service\ + import (PythonService, PythonServiceHandler) +from tabpy.tabpy_server.handlers\ + import (EndpointHandler, EndpointsHandler, + EvaluationPlaneHandler, QueryPlaneHandler, + ServiceInfoHandler, StatusHandler, + UploadDestinationHandler) import tornado @@ -171,28 +174,37 @@ def set_parameter(settings_key, config_key, default_val=None, check_env_var=False): + key_is_set = False + if config_key is not None and\ parser.has_section('TabPy') and\ parser.has_option('TabPy', config_key): self.settings[settings_key] = parser.get('TabPy', config_key) + key_is_set = True logger.debug( f'Parameter {settings_key} set to ' f'"{self.settings[settings_key]}" ' 'from config file') - elif check_env_var: - self.settings[settings_key] = os.getenv( - config_key, default_val) - logger.debug( - f'Parameter {settings_key} set to ' - f'"{self.settings[settings_key]}" ' - 'from environment variable') - elif default_val is not None: + + if not key_is_set and check_env_var: + val = os.getenv(config_key) + if val is not None: + self.settings[settings_key] = val + key_is_set = True + logger.debug( + f'Parameter {settings_key} set to ' + f'"{self.settings[settings_key]}" ' + 'from environment variable') + + if not key_is_set and default_val is not None: self.settings[settings_key] = default_val + key_is_set = True logger.debug( f'Parameter {settings_key} set to ' f'"{self.settings[settings_key]}" ' 'from default value') - else: + + if not key_is_set: logger.debug( f'Parameter {settings_key} is not set') @@ -213,9 +225,12 @@ def set_parameter(settings_key, 'to evaluate timeout of 30 seconds.') self.settings[SettingsParameters.EvaluateTimeout] = 30 + pkg_path = os.path.dirname(tabpy.__file__) set_parameter(SettingsParameters.UploadDir, ConfigParameters.TABPY_QUERY_OBJECT_PATH, - default_val='/tmp/query_objects', check_env_var=True) + default_val=os.path.join(pkg_path, + 'tmp', 'query_objects'), + check_env_var=True) if not os.path.exists(self.settings[SettingsParameters.UploadDir]): os.makedirs(self.settings[SettingsParameters.UploadDir]) @@ -236,16 +251,23 @@ def set_parameter(settings_key, # last dependence on batch/shell script set_parameter(SettingsParameters.StateFilePath, ConfigParameters.TABPY_STATE_PATH, - default_val='./tabpy-server/tabpy_server', + default_val=os.path.join(pkg_path, 'tabpy_server'), check_env_var=True) self.settings[SettingsParameters.StateFilePath] = os.path.realpath( os.path.normpath( os.path.expanduser( self.settings[SettingsParameters.StateFilePath]))) - state_file_path = self.settings[SettingsParameters.StateFilePath] - logger.info('Loading state from state file ' - f'{os.path.join(state_file_path, "state.ini")}') - tabpy_state = _get_state_from_file(state_file_path) + state_file_dir = self.settings[SettingsParameters.StateFilePath] + state_file_path = os.path.join(state_file_dir, 'state.ini') + if not os.path.isfile(state_file_path): + state_file_template_path = os.path.join( + pkg_path, 'tabpy_server', 'state.ini.template') + logger.debug(f'File {state_file_path} not found, creating from ' + f'template {state_file_template_path}...') + shutil.copy(state_file_template_path, state_file_path) + + logger.info(f'Loading state from state file {state_file_path}') + tabpy_state = _get_state_from_file(state_file_dir) self.tabpy_state = TabPyState( config=tabpy_state, settings=self.settings) @@ -320,7 +342,7 @@ def _validate_transfer_protocol_settings(self): 'an existing file.', os.path.isfile(cert), os.path.isfile(self.settings[SettingsParameters.KeyFile])) - tabpy_server.app.util.validate_cert(cert) + tabpy.tabpy_server.app.util.validate_cert(cert) @staticmethod def _validate_cert_key_state(msg, cert_valid, key_valid): diff --git a/tabpy-server/tabpy_server/app/util.py b/tabpy/tabpy_server/app/util.py similarity index 100% rename from tabpy-server/tabpy_server/app/util.py rename to tabpy/tabpy_server/app/util.py diff --git a/tabpy/tabpy_server/common/__init__.py b/tabpy/tabpy_server/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tabpy-server/tabpy_server/common/default.conf b/tabpy/tabpy_server/common/default.conf similarity index 79% rename from tabpy-server/tabpy_server/common/default.conf rename to tabpy/tabpy_server/common/default.conf index 5dd79d53..52786491 100755 --- a/tabpy-server/tabpy_server/common/default.conf +++ b/tabpy/tabpy_server/common/default.conf @@ -1,20 +1,20 @@ [TabPy] -TABPY_QUERY_OBJECT_PATH = /tmp/query_objects -TABPY_PORT = 9004 -TABPY_STATE_PATH = ./tabpy-server/tabpy_server +# TABPY_QUERY_OBJECT_PATH = /tmp/query_objects +# TABPY_PORT = 9004 +# TABPY_STATE_PATH = ./tabpy/tabpy_server # Where static pages live -TABPY_STATIC_PATH = ./tabpy-server/tabpy_server/static +# TABPY_STATIC_PATH = ./tabpy/tabpy_server/static # For how to configure TabPy authentication read -# docs/server-config.md. +# Authentication section in docs/server-config.md. # TABPY_PWD_FILE = /path/to/password/file.txt # To set up secure TabPy uncomment and modify the following lines. # Note only PEM-encoded x509 certificates are supported. # TABPY_TRANSFER_PROTOCOL = https -# TABPY_CERTIFICATE_FILE = path/to/certificate/file.crt -# TABPY_KEY_FILE = path/to/key/file.key +# TABPY_CERTIFICATE_FILE = /path/to/certificate/file.crt +# TABPY_KEY_FILE = /path/to/key/file.key # Log additional request details including caller IP, full URL, client # end user info if provided. diff --git a/tabpy-server/tabpy_server/common/endpoint_file_mgr.py b/tabpy/tabpy_server/common/endpoint_file_mgr.py similarity index 100% rename from tabpy-server/tabpy_server/common/endpoint_file_mgr.py rename to tabpy/tabpy_server/common/endpoint_file_mgr.py diff --git a/tabpy-server/tabpy_server/common/messages.py b/tabpy/tabpy_server/common/messages.py similarity index 100% rename from tabpy-server/tabpy_server/common/messages.py rename to tabpy/tabpy_server/common/messages.py diff --git a/tabpy-server/tabpy_server/common/util.py b/tabpy/tabpy_server/common/util.py similarity index 100% rename from tabpy-server/tabpy_server/common/util.py rename to tabpy/tabpy_server/common/util.py diff --git a/tabpy/tabpy_server/handlers/__init__.py b/tabpy/tabpy_server/handlers/__init__.py new file mode 100644 index 00000000..c73ea4d8 --- /dev/null +++ b/tabpy/tabpy_server/handlers/__init__.py @@ -0,0 +1,13 @@ +from tabpy.tabpy_server.handlers.base_handler import BaseHandler +from tabpy.tabpy_server.handlers.main_handler import MainHandler +from tabpy.tabpy_server.handlers.management_handler import ManagementHandler + +from tabpy.tabpy_server.handlers.endpoint_handler import EndpointHandler +from tabpy.tabpy_server.handlers.endpoints_handler import EndpointsHandler +from tabpy.tabpy_server.handlers.evaluation_plane_handler\ + import EvaluationPlaneHandler +from tabpy.tabpy_server.handlers.query_plane_handler import QueryPlaneHandler +from tabpy.tabpy_server.handlers.service_info_handler import ServiceInfoHandler +from tabpy.tabpy_server.handlers.status_handler import StatusHandler +from tabpy.tabpy_server.handlers.upload_destination_handler\ + import UploadDestinationHandler diff --git a/tabpy-server/tabpy_server/handlers/base_handler.py b/tabpy/tabpy_server/handlers/base_handler.py similarity index 99% rename from tabpy-server/tabpy_server/handlers/base_handler.py rename to tabpy/tabpy_server/handlers/base_handler.py index 3b22f7dc..4ee0934d 100644 --- a/tabpy-server/tabpy_server/handlers/base_handler.py +++ b/tabpy/tabpy_server/handlers/base_handler.py @@ -4,8 +4,8 @@ import json import logging import tornado.web -from tabpy_server.app.SettingsParameters import SettingsParameters -from tabpy_server.handlers.util import hash_password +from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters +from tabpy.tabpy_server.handlers.util import hash_password import uuid diff --git a/tabpy-server/tabpy_server/handlers/endpoint_handler.py b/tabpy/tabpy_server/handlers/endpoint_handler.py similarity index 93% rename from tabpy-server/tabpy_server/handlers/endpoint_handler.py rename to tabpy/tabpy_server/handlers/endpoint_handler.py index cfbf5957..20b6b334 100644 --- a/tabpy-server/tabpy_server/handlers/endpoint_handler.py +++ b/tabpy/tabpy_server/handlers/endpoint_handler.py @@ -6,17 +6,17 @@ at endpoints_handler.py ''' -from tabpy_server.handlers import ManagementHandler +import concurrent import json import logging -import tornado.web -from tornado import gen -from tabpy_server.management.state import get_query_object_path -from tabpy_server.common.util import format_exception -from tabpy_server.handlers.base_handler import STAGING_THREAD -from tabpy_server.psws.callbacks import on_state_change -import concurrent import shutil +from tabpy.tabpy_server.common.util import format_exception +from tabpy.tabpy_server.handlers import ManagementHandler +from tabpy.tabpy_server.handlers.base_handler import STAGING_THREAD +from tabpy.tabpy_server.management.state import get_query_object_path +from tabpy.tabpy_server.psws.callbacks import on_state_change +from tornado import gen +import tornado.web class EndpointHandler(ManagementHandler): diff --git a/tabpy-server/tabpy_server/handlers/endpoints_handler.py b/tabpy/tabpy_server/handlers/endpoints_handler.py similarity index 95% rename from tabpy-server/tabpy_server/handlers/endpoints_handler.py rename to tabpy/tabpy_server/handlers/endpoints_handler.py index 8745e457..bd54311b 100644 --- a/tabpy-server/tabpy_server/handlers/endpoints_handler.py +++ b/tabpy/tabpy_server/handlers/endpoints_handler.py @@ -6,12 +6,12 @@ at endpoint_handler.py ''' -from tabpy_server.handlers import ManagementHandler import json -import tornado.web -from tornado import gen import logging -from tabpy_server.common.util import format_exception +from tabpy.tabpy_server.common.util import format_exception +from tabpy.tabpy_server.handlers import ManagementHandler +from tornado import gen +import tornado.web class EndpointsHandler(ManagementHandler): diff --git a/tabpy-server/tabpy_server/handlers/evaluation_plane_handler.py b/tabpy/tabpy_server/handlers/evaluation_plane_handler.py similarity index 97% rename from tabpy-server/tabpy_server/handlers/evaluation_plane_handler.py rename to tabpy/tabpy_server/handlers/evaluation_plane_handler.py index 2a5686b2..f3d799f6 100644 --- a/tabpy-server/tabpy_server/handlers/evaluation_plane_handler.py +++ b/tabpy/tabpy_server/handlers/evaluation_plane_handler.py @@ -1,7 +1,7 @@ -from tabpy_server.handlers import BaseHandler +from tabpy.tabpy_server.handlers import BaseHandler import json import logging -from tabpy_server.common.util import format_exception +from tabpy.tabpy_server.common.util import format_exception import requests from tornado import gen from datetime import timedelta diff --git a/tabpy-server/tabpy_server/handlers/main_handler.py b/tabpy/tabpy_server/handlers/main_handler.py similarity index 70% rename from tabpy-server/tabpy_server/handlers/main_handler.py rename to tabpy/tabpy_server/handlers/main_handler.py index b43cb8d7..9961da2e 100644 --- a/tabpy-server/tabpy_server/handlers/main_handler.py +++ b/tabpy/tabpy_server/handlers/main_handler.py @@ -1,4 +1,4 @@ -from tabpy_server.handlers import BaseHandler +from tabpy.tabpy_server.handlers import BaseHandler class MainHandler(BaseHandler): diff --git a/tabpy-server/tabpy_server/handlers/management_handler.py b/tabpy/tabpy_server/handlers/management_handler.py similarity index 94% rename from tabpy-server/tabpy_server/handlers/management_handler.py rename to tabpy/tabpy_server/handlers/management_handler.py index 972876f3..805d3e51 100644 --- a/tabpy-server/tabpy_server/handlers/management_handler.py +++ b/tabpy/tabpy_server/handlers/management_handler.py @@ -8,11 +8,11 @@ from tornado import gen -from tabpy_server.app.SettingsParameters import SettingsParameters -from tabpy_server.handlers import MainHandler -from tabpy_server.handlers.base_handler import STAGING_THREAD -from tabpy_server.management.state import get_query_object_path -from tabpy_server.psws.callbacks import on_state_change +from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters +from tabpy.tabpy_server.handlers import MainHandler +from tabpy.tabpy_server.handlers.base_handler import STAGING_THREAD +from tabpy.tabpy_server.management.state import get_query_object_path +from tabpy.tabpy_server.psws.callbacks import on_state_change def copy_from_local(localpath, remotepath, is_dir=False): diff --git a/tabpy-server/tabpy_server/handlers/query_plane_handler.py b/tabpy/tabpy_server/handlers/query_plane_handler.py similarity index 97% rename from tabpy-server/tabpy_server/handlers/query_plane_handler.py rename to tabpy/tabpy_server/handlers/query_plane_handler.py index 1950cbf7..70774626 100644 --- a/tabpy-server/tabpy_server/handlers/query_plane_handler.py +++ b/tabpy/tabpy_server/handlers/query_plane_handler.py @@ -1,12 +1,12 @@ -from tabpy_server.handlers import BaseHandler +from tabpy.tabpy_server.handlers import BaseHandler import logging import time -from tabpy_server.common.messages import ( +from tabpy.tabpy_server.common.messages import ( Query, QuerySuccessful, QueryError, UnknownURI) from hashlib import md5 import uuid import json -from tabpy_server.common.util import format_exception +from tabpy.tabpy_server.common.util import format_exception import urllib import tornado.web from tornado import gen diff --git a/tabpy-server/tabpy_server/handlers/service_info_handler.py b/tabpy/tabpy_server/handlers/service_info_handler.py similarity index 86% rename from tabpy-server/tabpy_server/handlers/service_info_handler.py rename to tabpy/tabpy_server/handlers/service_info_handler.py index 1d1c3059..6341c149 100644 --- a/tabpy-server/tabpy_server/handlers/service_info_handler.py +++ b/tabpy/tabpy_server/handlers/service_info_handler.py @@ -1,6 +1,6 @@ import json -from tabpy_server.app.SettingsParameters import SettingsParameters -from tabpy_server.handlers import ManagementHandler +from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters +from tabpy.tabpy_server.handlers import ManagementHandler class ServiceInfoHandler(ManagementHandler): diff --git a/tabpy-server/tabpy_server/handlers/status_handler.py b/tabpy/tabpy_server/handlers/status_handler.py similarity index 93% rename from tabpy-server/tabpy_server/handlers/status_handler.py rename to tabpy/tabpy_server/handlers/status_handler.py index 60dd9a70..3b2af815 100644 --- a/tabpy-server/tabpy_server/handlers/status_handler.py +++ b/tabpy/tabpy_server/handlers/status_handler.py @@ -1,6 +1,6 @@ import json import logging -from tabpy_server.handlers import BaseHandler +from tabpy.tabpy_server.handlers import BaseHandler class StatusHandler(BaseHandler): diff --git a/tabpy-server/tabpy_server/handlers/upload_destination_handler.py b/tabpy/tabpy_server/handlers/upload_destination_handler.py similarity index 79% rename from tabpy-server/tabpy_server/handlers/upload_destination_handler.py rename to tabpy/tabpy_server/handlers/upload_destination_handler.py index f94ef630..5211b1e6 100644 --- a/tabpy-server/tabpy_server/handlers/upload_destination_handler.py +++ b/tabpy/tabpy_server/handlers/upload_destination_handler.py @@ -1,6 +1,6 @@ import logging -from tabpy_server.app.SettingsParameters import SettingsParameters -from tabpy_server.handlers import ManagementHandler +from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters +from tabpy.tabpy_server.handlers import ManagementHandler import os diff --git a/tabpy-server/tabpy_server/handlers/util.py b/tabpy/tabpy_server/handlers/util.py similarity index 89% rename from tabpy-server/tabpy_server/handlers/util.py rename to tabpy/tabpy_server/handlers/util.py index da7929e0..e835d7fc 100755 --- a/tabpy-server/tabpy_server/handlers/util.py +++ b/tabpy/tabpy_server/handlers/util.py @@ -1,7 +1,7 @@ import base64 import binascii from hashlib import pbkdf2_hmac -from tabpy_server.app.SettingsParameters import SettingsParameters +from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters def hash_password(username, pwd): diff --git a/tabpy/tabpy_server/management/__init__.py b/tabpy/tabpy_server/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tabpy-server/tabpy_server/management/state.py b/tabpy/tabpy_server/management/state.py similarity index 99% rename from tabpy-server/tabpy_server/management/state.py rename to tabpy/tabpy_server/management/state.py index c4ba9bb1..c1353ece 100644 --- a/tabpy-server/tabpy_server/management/state.py +++ b/tabpy/tabpy_server/management/state.py @@ -5,7 +5,7 @@ import json import logging import sys -from tabpy_server.management.util import write_state_config +from tabpy.tabpy_server.management.util import write_state_config from threading import Lock from time import time diff --git a/tabpy-server/tabpy_server/management/util.py b/tabpy/tabpy_server/management/util.py similarity index 93% rename from tabpy-server/tabpy_server/management/util.py rename to tabpy/tabpy_server/management/util.py index 5d93c63f..13d1eae0 100644 --- a/tabpy-server/tabpy_server/management/util.py +++ b/tabpy/tabpy_server/management/util.py @@ -5,8 +5,8 @@ except ImportError: from configparser import ConfigParser as _ConfigParser from datetime import datetime, timedelta, tzinfo -from tabpy_server.app.ConfigParameters import ConfigParameters -from tabpy_server.app.SettingsParameters import SettingsParameters +from tabpy.tabpy_server.app.ConfigParameters import ConfigParameters +from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters def write_state_config(state, settings, logger=logging.getLogger(__name__)): diff --git a/tabpy/tabpy_server/psws/__init__.py b/tabpy/tabpy_server/psws/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tabpy-server/tabpy_server/psws/callbacks.py b/tabpy/tabpy_server/psws/callbacks.py similarity index 93% rename from tabpy-server/tabpy_server/psws/callbacks.py rename to tabpy/tabpy_server/psws/callbacks.py index 903d9998..d6c38b4f 100644 --- a/tabpy-server/tabpy_server/psws/callbacks.py +++ b/tabpy/tabpy_server/psws/callbacks.py @@ -1,12 +1,13 @@ import logging import sys -from tabpy_server.app.SettingsParameters import SettingsParameters -from tabpy_server.common.messages import ( - LoadObject, DeleteObjects, ListObjects, ObjectList) -from tabpy_server.common.endpoint_file_mgr import cleanup_endpoint_files -from tabpy_server.common.util import format_exception -from tabpy_server.management.state import TabPyState, get_query_object_path -from tabpy_server.management import util +from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters +from tabpy.tabpy_server.common.messages\ + import (LoadObject, DeleteObjects, ListObjects, ObjectList) +from tabpy.tabpy_server.common.endpoint_file_mgr import cleanup_endpoint_files +from tabpy.tabpy_server.common.util import format_exception +from tabpy.tabpy_server.management.state\ + import TabPyState, get_query_object_path +from tabpy.tabpy_server.management import util from time import sleep from tornado import gen diff --git a/tabpy-server/tabpy_server/psws/python_service.py b/tabpy/tabpy_server/psws/python_service.py similarity index 98% rename from tabpy-server/tabpy_server/psws/python_service.py rename to tabpy/tabpy_server/psws/python_service.py index f8093623..4a64e810 100644 --- a/tabpy-server/tabpy_server/psws/python_service.py +++ b/tabpy/tabpy_server/psws/python_service.py @@ -3,9 +3,9 @@ import sys -from tabpy_tools.query_object import QueryObject -from tabpy_server.common.util import format_exception -from tabpy_server.common.messages import ( +from tabpy.tabpy_tools.query_object import QueryObject +from tabpy.tabpy_server.common.util import format_exception +from tabpy.tabpy_server.common.messages import ( LoadObject, DeleteObjects, FlushObjects, CountObjects, ListObjects, UnknownMessage, LoadFailed, ObjectsDeleted, ObjectsFlushed, QueryFailed, QuerySuccessful, UnknownURI, DownloadSkipped, LoadInProgress, ObjectCount, diff --git a/tabpy-server/tabpy_server/state.ini b/tabpy/tabpy_server/state.ini.template similarity index 86% rename from tabpy-server/tabpy_server/state.ini rename to tabpy/tabpy_server/state.ini.template index 36b1d2ac..b3828973 100755 --- a/tabpy-server/tabpy_server/state.ini +++ b/tabpy/tabpy_server/state.ini.template @@ -11,4 +11,5 @@ Access-Control-Allow-Methods = [Query Objects Docstrings] [Meta] -Revision Number = 1 \ No newline at end of file +Revision Number = 1 + diff --git a/tabpy-server/tabpy_server/static/index.html b/tabpy/tabpy_server/static/index.html similarity index 100% rename from tabpy-server/tabpy_server/static/index.html rename to tabpy/tabpy_server/static/index.html diff --git a/tabpy-server/tabpy_server/static/tableau.png b/tabpy/tabpy_server/static/tableau.png similarity index 100% rename from tabpy-server/tabpy_server/static/tableau.png rename to tabpy/tabpy_server/static/tableau.png diff --git a/tabpy/tabpy_tools/__init__.py b/tabpy/tabpy_tools/__init__.py new file mode 100755 index 00000000..e69de29b diff --git a/tabpy-tools/tabpy_tools/client.py b/tabpy/tabpy_tools/client.py similarity index 100% rename from tabpy-tools/tabpy_tools/client.py rename to tabpy/tabpy_tools/client.py diff --git a/tabpy-tools/tabpy_tools/custom_query_object.py b/tabpy/tabpy_tools/custom_query_object.py similarity index 100% rename from tabpy-tools/tabpy_tools/custom_query_object.py rename to tabpy/tabpy_tools/custom_query_object.py diff --git a/tabpy-tools/tabpy_tools/query_object.py b/tabpy/tabpy_tools/query_object.py similarity index 100% rename from tabpy-tools/tabpy_tools/query_object.py rename to tabpy/tabpy_tools/query_object.py diff --git a/tabpy-tools/tabpy_tools/rest.py b/tabpy/tabpy_tools/rest.py similarity index 100% rename from tabpy-tools/tabpy_tools/rest.py rename to tabpy/tabpy_tools/rest.py diff --git a/tabpy-tools/tabpy_tools/rest_client.py b/tabpy/tabpy_tools/rest_client.py similarity index 100% rename from tabpy-tools/tabpy_tools/rest_client.py rename to tabpy/tabpy_tools/rest_client.py diff --git a/tabpy-tools/tabpy_tools/schema.py b/tabpy/tabpy_tools/schema.py similarity index 98% rename from tabpy-tools/tabpy_tools/schema.py rename to tabpy/tabpy_tools/schema.py index f001d7f0..080d3529 100755 --- a/tabpy-tools/tabpy_tools/schema.py +++ b/tabpy/tabpy_tools/schema.py @@ -1,7 +1,6 @@ import logging import genson as _genson - -from json import validate as _validate +import jsonschema logger = logging.getLogger(__name__) @@ -44,7 +43,7 @@ def _generate_schema_from_example_and_description(input, description): try: # This should not fail unless there are bugs with either genson or # jsonschema. - _validate(input, input_schema) + jsonschema.validate(input, input_schema) except Exception as e: logger.error(f'Internal error validating schema: {str(e)}') raise diff --git a/tabpy/utils/__init__.py b/tabpy/utils/__init__.py new file mode 100755 index 00000000..e69de29b diff --git a/utils/user_management.py b/tabpy/utils/user_management.py similarity index 94% rename from utils/user_management.py rename to tabpy/utils/user_management.py index 45676f93..774af11c 100755 --- a/utils/user_management.py +++ b/tabpy/utils/user_management.py @@ -7,8 +7,8 @@ import os import secrets import sys -from tabpy_server.app.util import parse_pwd_file -from tabpy_server.handlers.util import hash_password +from tabpy.tabpy_server.app.util import parse_pwd_file +from tabpy.tabpy_server.handlers.util import hash_password logger = logging.getLogger(__name__) @@ -132,6 +132,8 @@ def process_command(args, credentials): def main(): + logging.basicConfig(level=logging.DEBUG, format="%(message)s") + parser = build_cli_parser() args = parser.parse_args() if not check_args(args): @@ -150,6 +152,4 @@ def main(): if __name__ == '__main__': - logging.basicConfig(level=logging.DEBUG, format="%(message)s") - main() diff --git a/tests/integration/integ_test_base.py b/tests/integration/integ_test_base.py index 11b0b2a9..13305642 100755 --- a/tests/integration/integ_test_base.py +++ b/tests/integration/integ_test_base.py @@ -1,9 +1,12 @@ +import coverage import http.client import os +from pathlib import Path import platform import shutil import signal import subprocess +import tabpy import tempfile import time import unittest @@ -222,17 +225,19 @@ def setUp(self): # Platform specific - for integration tests we want to engage # startup script with open(self.tmp_dir + '/output.txt', 'w') as outfile: + cmd = ['tabpy', + '--config=' + self.config_file_name] + coverage.process_startup() if platform.system() == 'Windows': self.py = 'python' self.process = subprocess.Popen( - ['startup.cmd', self.config_file_name], + cmd, stdout=outfile, stderr=outfile) else: self.py = 'python3' self.process = subprocess.Popen( - ['./startup.sh', - '--config=' + self.config_file_name], + cmd, preexec_fn=os.setsid, stdout=outfile, stderr=outfile) @@ -272,3 +277,25 @@ def _get_connection(self) -> http.client.HTTPConnection: connection = http.client.HTTPConnection(url) return connection + + def _get_username(self) -> str: + return 'user1' + + def _get_password(self) -> str: + return 'P@ssw0rd' + + def deploy_models(self, username: str, password: str): + repo_dir = os.path.abspath(os.path.dirname(tabpy.__file__)) + path = os.path.join(repo_dir, 'models', 'deploy_models.py') + with open(self.tmp_dir + '/deploy_models_output.txt', 'w') as outfile: + outfile.write( + f'--<< Running {self.py} {path} ' + f'{self._get_config_file_name()} >>--\n') + input_string = f'{username}\n{password}\n' + outfile.write(f'--<< Input = {input_string} >>--') + coverage.process_startup() + p = subprocess.run( + [self.py, path, self._get_config_file_name()], + input=input_string.encode('utf-8'), + stdout=outfile, + stderr=outfile) diff --git a/tests/integration/resources/deploy_and_evaluate_model.conf b/tests/integration/resources/deploy_and_evaluate_model.conf new file mode 100755 index 00000000..375ea47a --- /dev/null +++ b/tests/integration/resources/deploy_and_evaluate_model.conf @@ -0,0 +1,57 @@ +[TabPy] +# TABPY_QUERY_OBJECT_PATH = /tmp/query_objects +TABPY_PORT = 9008 +# TABPY_STATE_PATH = ./tabpy/tabpy_server + +# Where static pages live +# TABPY_STATIC_PATH = ./tabpy/tabpy_server/static + +# For how to configure TabPy authentication read +# Authentication section in docs/server-config.md. +# TABPY_PWD_FILE = /path/to/password/file.txt + +# To set up secure TabPy uncomment and modify the following lines. +# Note only PEM-encoded x509 certificates are supported. +# TABPY_TRANSFER_PROTOCOL = https +# TABPY_CERTIFICATE_FILE = path/to/certificate/file.crt +# TABPY_KEY_FILE = path/to/key/file.key + +# Log additional request details including caller IP, full URL, client +# end user info if provided. +# TABPY_LOG_DETAILS = true + +# Configure how long a custom script provided to the /evaluate method +# will run before throwing a TimeoutError. +# The value should be a float representing the timeout time in seconds. +#TABPY_EVALUATE_TIMEOUT = 30 + +[loggers] +keys=root + +[handlers] +keys=rootHandler,rotatingFileHandler + +[formatters] +keys=rootFormatter + +[logger_root] +level=DEBUG +handlers=rootHandler,rotatingFileHandler +qualname=root +propagete=0 + +[handler_rootHandler] +class=StreamHandler +level=DEBUG +formatter=rootFormatter +args=(sys.stdout,) + +[handler_rotatingFileHandler] +class=handlers.RotatingFileHandler +level=DEBUG +formatter=rootFormatter +args=('tabpy_log.log', 'a', 1000000, 5) + +[formatter_rootFormatter] +format=%(asctime)s [%(levelname)s] (%(filename)s:%(module)s:%(lineno)d): %(message)s +datefmt=%Y-%m-%d,%H:%M:%S diff --git a/tests/integration/test_deploy_and_evaluate_model.py b/tests/integration/test_deploy_and_evaluate_model.py index 57f1bb20..3a2e80f5 100644 --- a/tests/integration/test_deploy_and_evaluate_model.py +++ b/tests/integration/test_deploy_and_evaluate_model.py @@ -4,9 +4,19 @@ class TestDeployAndEvaluateModel(integ_test_base.IntegTestBase): + def _get_config_file_name(self) -> str: + return './tests/integration/resources/deploy_and_evaluate_model.conf' + + def _get_port(self) -> str: + return '9008' + def test_deploy_and_evaluate_model(self): - path = str(Path('models', 'setup.py')) - subprocess.call([self.py, path, self._get_config_file_name()]) + # Uncomment the following line to preserve + # test case output and other files (config, state, ect.) + # in system temp folder. + self.set_delete_temp_folder(False) + + self.deploy_models(self._get_username(), self._get_password()) payload = ( '''{ diff --git a/tests/integration/test_deploy_and_evaluate_model_ssl.py b/tests/integration/test_deploy_and_evaluate_model_ssl.py index 669dc724..09a68feb 100755 --- a/tests/integration/test_deploy_and_evaluate_model_ssl.py +++ b/tests/integration/test_deploy_and_evaluate_model_ssl.py @@ -18,8 +18,7 @@ def _get_key_file_name(self) -> str: return './tests/integration/resources/2019_04_24_to_3018_08_25.key' def test_deploy_and_evaluate_model_ssl(self): - path = str(Path('models', 'setup.py')) - subprocess.call([self.py, path, self._get_config_file_name()]) + self.deploy_models(self._get_username(), self._get_password()) payload = ( '''{ diff --git a/tests/integration/test_deploy_model_ssl_off_auth_off.py b/tests/integration/test_deploy_model_ssl_off_auth_off.py index e35d81d8..f5b0749d 100644 --- a/tests/integration/test_deploy_model_ssl_off_auth_off.py +++ b/tests/integration/test_deploy_model_ssl_off_auth_off.py @@ -5,11 +5,11 @@ class TestDeployModelSSLOffAuthOff(integ_test_base.IntegTestBase): def test_deploy_ssl_off_auth_off(self): - models = ['PCA', 'Sentiment%20Analysis', "ttest"] - path = str(Path('models', 'setup.py')) - subprocess.call([self.py, path, self._get_config_file_name()]) + self.deploy_models(self._get_username(), self._get_password()) conn = self._get_connection() + + models = ['PCA', 'Sentiment%20Analysis', "ttest"] for m in models: conn.request("GET", f'/endpoints/{m}') m_request = conn.getresponse() diff --git a/tests/integration/test_deploy_model_ssl_off_auth_on.py b/tests/integration/test_deploy_model_ssl_off_auth_on.py index bb1268eb..0f09bdc6 100644 --- a/tests/integration/test_deploy_model_ssl_off_auth_on.py +++ b/tests/integration/test_deploy_model_ssl_off_auth_on.py @@ -9,10 +9,7 @@ def _get_pwd_file(self) -> str: return './tests/integration/resources/pwdfile.txt' def test_deploy_ssl_off_auth_on(self): - models = ['PCA', 'Sentiment%20Analysis', "ttest"] - path = str(Path('models', 'setup.py')) - p = subprocess.run([self.py, path, self._get_config_file_name()], - input=b'user1\nP@ssw0rd\n') + self.deploy_models(self._get_username(), self._get_password()) headers = { 'Content-Type': "application/json", @@ -24,6 +21,8 @@ def test_deploy_ssl_off_auth_on(self): } conn = self._get_connection() + + models = ['PCA', 'Sentiment%20Analysis', "ttest"] for m in models: conn.request("GET", f'/endpoints/{m}', headers=headers) m_request = conn.getresponse() diff --git a/tests/integration/test_deploy_model_ssl_on_auth_off.py b/tests/integration/test_deploy_model_ssl_on_auth_off.py index fe083849..8c549b47 100644 --- a/tests/integration/test_deploy_model_ssl_on_auth_off.py +++ b/tests/integration/test_deploy_model_ssl_on_auth_off.py @@ -15,9 +15,7 @@ def _get_key_file_name(self) -> str: return './tests/integration/resources/2019_04_24_to_3018_08_25.key' def test_deploy_ssl_on_auth_off(self): - models = ['PCA', 'Sentiment%20Analysis', "ttest"] - path = str(Path('models', 'setup.py')) - subprocess.call([self.py, path, self._get_config_file_name()]) + self.deploy_models(self._get_username(), self._get_password()) session = requests.Session() # Do not verify servers' cert to be signed by trusted CA @@ -25,6 +23,7 @@ def test_deploy_ssl_on_auth_off(self): # Do not warn about insecure request requests.packages.urllib3.disable_warnings() + models = ['PCA', 'Sentiment%20Analysis', "ttest"] for m in models: m_response = session.get(url=f'{self._get_transfer_protocol()}://' f'localhost:9004/endpoints/{m}') diff --git a/tests/integration/test_deploy_model_ssl_on_auth_on.py b/tests/integration/test_deploy_model_ssl_on_auth_on.py index 742abceb..142d6cde 100644 --- a/tests/integration/test_deploy_model_ssl_on_auth_on.py +++ b/tests/integration/test_deploy_model_ssl_on_auth_on.py @@ -2,7 +2,6 @@ import base64 import requests import subprocess -from pathlib import Path class TestDeployModelSSLOnAuthOn(integ_test_base.IntegTestBase): @@ -19,10 +18,12 @@ def _get_pwd_file(self) -> str: return './tests/integration/resources/pwdfile.txt' def test_deploy_ssl_on_auth_on(self): - models = ['PCA', 'Sentiment%20Analysis', "ttest"] - path = str(Path('models', 'setup.py')) - p = subprocess.run([self.py, path, self._get_config_file_name()], - input=b'user1\nP@ssw0rd\n') + # Uncomment the following line to preserve + # test case output and other files (config, state, ect.) + # in system temp folder. + # self.set_delete_temp_folder(False) + + self.deploy_models(self._get_username(), self._get_password()) headers = { 'Content-Type': "application/json", @@ -37,6 +38,7 @@ def test_deploy_ssl_on_auth_on(self): # Do not warn about insecure request requests.packages.urllib3.disable_warnings() + models = ['PCA', 'Sentiment%20Analysis', "ttest"] for m in models: m_response = session.get(url=f'{self._get_transfer_protocol()}://' f'localhost:9004/endpoints/{m}', diff --git a/tests/unit/server_tests/test_config.py b/tests/unit/server_tests/test_config.py index 376cc98a..6d41ec36 100644 --- a/tests/unit/server_tests/test_config.py +++ b/tests/unit/server_tests/test_config.py @@ -2,9 +2,9 @@ import unittest from argparse import Namespace from tempfile import NamedTemporaryFile - -from tabpy_server.app.util import validate_cert -from tabpy_server.app.app import TabPyApp +import tabpy +from tabpy.tabpy_server.app.util import validate_cert +from tabpy.tabpy_server.app.app import TabPyApp from unittest.mock import patch, call @@ -18,41 +18,44 @@ def assert_raises_runtime_error(message, fn, args={}): class TestConfigEnvironmentCalls(unittest.TestCase): - @patch('tabpy_server.app.app.TabPyApp._parse_cli_arguments', + @patch('tabpy.tabpy_server.app.app.TabPyApp._parse_cli_arguments', return_value=Namespace(config=None)) - @patch('tabpy_server.app.app.TabPyState') - @patch('tabpy_server.app.app._get_state_from_file') - @patch('tabpy_server.app.app.PythonServiceHandler') - @patch('tabpy_server.app.app.os.path.exists', return_value=True) - @patch('tabpy_server.app.app.os.path.isfile', return_value=False) - @patch('tabpy_server.app.app.os') - def test_no_config_file(self, mock_os, mock_file_exists, + @patch('tabpy.tabpy_server.app.app.TabPyState') + @patch('tabpy.tabpy_server.app.app._get_state_from_file') + @patch('tabpy.tabpy_server.app.app.PythonServiceHandler') + @patch('tabpy.tabpy_server.app.app.os.path.exists', return_value=True) + @patch('tabpy.tabpy_server.app.app.os') + def test_no_config_file(self, mock_os, mock_path_exists, mock_psws, mock_management_util, mock_tabpy_state, mock_parse_arguments): + pkg_path = os.path.dirname(tabpy.__file__) + obj_path = os.path.join(pkg_path, 'tmp', 'query_objects') + state_path = os.path.join(pkg_path, 'tabpy_server') + + mock_os.getenv.side_effect = [9004, obj_path, state_path] + TabPyApp(None) - getenv_calls = [call('TABPY_PORT', 9004), - call('TABPY_QUERY_OBJECT_PATH', '/tmp/query_objects'), - call('TABPY_STATE_PATH', - './tabpy-server/tabpy_server')] + getenv_calls = [ + call('TABPY_PORT'), + call('TABPY_QUERY_OBJECT_PATH'), + call('TABPY_STATE_PATH')] mock_os.getenv.assert_has_calls(getenv_calls, any_order=True) - self.assertEqual(len(mock_file_exists.mock_calls), 2) self.assertEqual(len(mock_psws.mock_calls), 1) self.assertEqual(len(mock_tabpy_state.mock_calls), 1) self.assertEqual(len(mock_path_exists.mock_calls), 1) self.assertTrue(len(mock_management_util.mock_calls) > 0) mock_os.makedirs.assert_not_called() - @patch('tabpy_server.app.app.TabPyApp._parse_cli_arguments', + @patch('tabpy.tabpy_server.app.app.TabPyApp._parse_cli_arguments', return_value=Namespace(config=None)) - @patch('tabpy_server.app.app.TabPyState') - @patch('tabpy_server.app.app._get_state_from_file') - @patch('tabpy_server.app.app.PythonServiceHandler') - @patch('tabpy_server.app.app.os.path.exists', return_value=False) - @patch('tabpy_server.app.app.os.path.isfile', return_value=False) - @patch('tabpy_server.app.app.os') - def test_no_state_ini_file_or_state_dir(self, mock_os, mock_file_exists, + @patch('tabpy.tabpy_server.app.app.TabPyState') + @patch('tabpy.tabpy_server.app.app._get_state_from_file') + @patch('tabpy.tabpy_server.app.app.PythonServiceHandler') + @patch('tabpy.tabpy_server.app.app.os.path.exists', return_value=False) + @patch('tabpy.tabpy_server.app.app.os') + def test_no_state_ini_file_or_state_dir(self, mock_os, mock_path_exists, mock_psws, mock_management_util, mock_tabpy_state, @@ -69,12 +72,12 @@ def tearDown(self): os.remove(self.config_file.name) self.config_file = None - @patch('tabpy_server.app.app.TabPyApp._parse_cli_arguments') - @patch('tabpy_server.app.app.TabPyState') - @patch('tabpy_server.app.app._get_state_from_file') - @patch('tabpy_server.app.app.PythonServiceHandler') - @patch('tabpy_server.app.app.os.path.exists', return_value=True) - @patch('tabpy_server.app.app.os') + @patch('tabpy.tabpy_server.app.app.TabPyApp._parse_cli_arguments') + @patch('tabpy.tabpy_server.app.app.TabPyState') + @patch('tabpy.tabpy_server.app.app._get_state_from_file') + @patch('tabpy.tabpy_server.app.app.PythonServiceHandler') + @patch('tabpy.tabpy_server.app.app.os.path.exists', return_value=True) + @patch('tabpy.tabpy_server.app.app.os') def test_config_file_present(self, mock_os, mock_path_exists, mock_psws, mock_management_util, mock_tabpy_state, mock_parse_arguments): @@ -91,7 +94,7 @@ def test_config_file_present(self, mock_os, mock_path_exists, mock_os.path.realpath.return_value = 'bar' app = TabPyApp(config_file.name) - getenv_calls = [call('TABPY_PORT', 9004)] + getenv_calls = [call('TABPY_PORT')] mock_os.getenv.assert_has_calls(getenv_calls, any_order=True) self.assertEqual(app.settings['port'], 1234) @@ -105,9 +108,9 @@ def test_config_file_present(self, mock_os, mock_path_exists, self.assertEqual(app.settings['log_request_context'], False) self.assertEqual(app.settings['evaluate_timeout'], 30) - @patch('tabpy_server.app.app.os.path.exists', return_value=True) - @patch('tabpy_server.app.app._get_state_from_file') - @patch('tabpy_server.app.app.TabPyState') + @patch('tabpy.tabpy_server.app.app.os.path.exists', return_value=True) + @patch('tabpy.tabpy_server.app.app._get_state_from_file') + @patch('tabpy.tabpy_server.app.app.TabPyState') def test_custom_evaluate_timeout_valid(self, mock_state, mock_get_state_from_file, mock_path_exists): @@ -120,9 +123,9 @@ def test_custom_evaluate_timeout_valid(self, mock_state, app = TabPyApp(self.config_file.name) self.assertEqual(app.settings['evaluate_timeout'], 1996.0) - @patch('tabpy_server.app.app.os.path.exists', return_value=True) - @patch('tabpy_server.app.app._get_state_from_file') - @patch('tabpy_server.app.app.TabPyState') + @patch('tabpy.tabpy_server.app.app.os.path.exists', return_value=True) + @patch('tabpy.tabpy_server.app.app._get_state_from_file') + @patch('tabpy.tabpy_server.app.app.TabPyState') def test_custom_evaluate_timeout_invalid(self, mock_state, mock_get_state_from_file, mock_path_exists): @@ -197,7 +200,7 @@ def test_https_without_key(self): 'TABPY_KEY_FILE must be set.', TabPyApp, {self.fp.name}) - @patch('tabpy_server.app.app.os.path') + @patch('tabpy.tabpy_server.app.app.os.path') def test_https_cert_and_key_file_not_found(self, mock_path): self.fp.write("[TabPy]\n" "TABPY_TRANSFER_PROTOCOL = https\n" @@ -213,7 +216,7 @@ def test_https_cert_and_key_file_not_found(self, mock_path): 'TABPY_KEY_FILE must point to an existing file.', TabPyApp, {self.fp.name}) - @patch('tabpy_server.app.app.os.path') + @patch('tabpy.tabpy_server.app.app.os.path') def test_https_cert_file_not_found(self, mock_path): self.fp.write("[TabPy]\n" "TABPY_TRANSFER_PROTOCOL = https\n" @@ -229,7 +232,7 @@ def test_https_cert_file_not_found(self, mock_path): 'must point to an existing file.', TabPyApp, {self.fp.name}) - @patch('tabpy_server.app.app.os.path') + @patch('tabpy.tabpy_server.app.app.os.path') def test_https_key_file_not_found(self, mock_path): self.fp.write("[TabPy]\n" "TABPY_TRANSFER_PROTOCOL = https\n" @@ -245,8 +248,8 @@ def test_https_key_file_not_found(self, mock_path): 'must point to an existing file.', TabPyApp, {self.fp.name}) - @patch('tabpy_server.app.app.os.path.isfile', return_value=True) - @patch('tabpy_server.app.util.validate_cert') + @patch('tabpy.tabpy_server.app.app.os.path.isfile', return_value=True) + @patch('tabpy.tabpy_server.app.util.validate_cert') def test_https_success(self, mock_isfile, mock_validate_cert): self.fp.write("[TabPy]\n" "TABPY_TRANSFER_PROTOCOL = HtTpS\n" diff --git a/tests/unit/server_tests/test_endpoint_file_manager.py b/tests/unit/server_tests/test_endpoint_file_manager.py index b87e9e6f..04ddb9cb 100644 --- a/tests/unit/server_tests/test_endpoint_file_manager.py +++ b/tests/unit/server_tests/test_endpoint_file_manager.py @@ -1,5 +1,5 @@ import unittest -from tabpy_server.common.endpoint_file_mgr import _check_endpoint_name +from tabpy.tabpy_server.common.endpoint_file_mgr import _check_endpoint_name class TestEndpointFileManager(unittest.TestCase): diff --git a/tests/unit/server_tests/test_endpoint_handler.py b/tests/unit/server_tests/test_endpoint_handler.py index 2f93e45d..f98135da 100755 --- a/tests/unit/server_tests/test_endpoint_handler.py +++ b/tests/unit/server_tests/test_endpoint_handler.py @@ -3,8 +3,8 @@ import tempfile from argparse import Namespace -from tabpy_server.app.app import TabPyApp -from tabpy_server.handlers.util import hash_password +from tabpy.tabpy_server.app.app import TabPyApp +from tabpy.tabpy_server.handlers.util import hash_password from tornado.testing import AsyncHTTPTestCase from unittest.mock import patch @@ -13,7 +13,7 @@ class TestEndpointHandlerWithAuth(AsyncHTTPTestCase): @classmethod def setUpClass(cls): cls.patcher = patch( - 'tabpy_server.app.app.TabPyApp._parse_cli_arguments', + 'tabpy.tabpy_server.app.app.TabPyApp._parse_cli_arguments', return_value=Namespace( config=None)) cls.patcher.start() diff --git a/tests/unit/server_tests/test_endpoints_handler.py b/tests/unit/server_tests/test_endpoints_handler.py index 096f1cf6..dcb422eb 100755 --- a/tests/unit/server_tests/test_endpoints_handler.py +++ b/tests/unit/server_tests/test_endpoints_handler.py @@ -3,8 +3,8 @@ import tempfile from argparse import Namespace -from tabpy_server.app.app import TabPyApp -from tabpy_server.handlers.util import hash_password +from tabpy.tabpy_server.app.app import TabPyApp +from tabpy.tabpy_server.handlers.util import hash_password from tornado.testing import AsyncHTTPTestCase from unittest.mock import patch @@ -13,7 +13,7 @@ class TestEndpointsHandlerWithAuth(AsyncHTTPTestCase): @classmethod def setUpClass(cls): cls.patcher = patch( - 'tabpy_server.app.app.TabPyApp._parse_cli_arguments', + 'tabpy.tabpy_server.app.app.TabPyApp._parse_cli_arguments', return_value=Namespace( config=None)) cls.patcher.start() diff --git a/tests/unit/server_tests/test_evaluation_plane_handler.py b/tests/unit/server_tests/test_evaluation_plane_handler.py index a02fed5b..45bf6962 100755 --- a/tests/unit/server_tests/test_evaluation_plane_handler.py +++ b/tests/unit/server_tests/test_evaluation_plane_handler.py @@ -3,8 +3,8 @@ import tempfile from argparse import Namespace -from tabpy_server.app.app import TabPyApp -from tabpy_server.handlers.util import hash_password +from tabpy.tabpy_server.app.app import TabPyApp +from tabpy.tabpy_server.handlers.util import hash_password from tornado.testing import AsyncHTTPTestCase from unittest.mock import patch @@ -13,7 +13,7 @@ class TestEvaluationPlainHandlerWithAuth(AsyncHTTPTestCase): @classmethod def setUpClass(cls): cls.patcher = patch( - 'tabpy_server.app.app.TabPyApp._parse_cli_arguments', + 'tabpy.tabpy_server.app.app.TabPyApp._parse_cli_arguments', return_value=Namespace( config=None)) cls.patcher.start() diff --git a/tests/unit/server_tests/test_pwd_file.py b/tests/unit/server_tests/test_pwd_file.py index 6c49194e..596ab9e2 100755 --- a/tests/unit/server_tests/test_pwd_file.py +++ b/tests/unit/server_tests/test_pwd_file.py @@ -2,7 +2,7 @@ import unittest from tempfile import NamedTemporaryFile -from tabpy_server.app.app import TabPyApp +from tabpy.tabpy_server.app.app import TabPyApp class TestPasswordFile(unittest.TestCase): diff --git a/tests/unit/server_tests/test_service_info_handler.py b/tests/unit/server_tests/test_service_info_handler.py index 1abf1399..9f61eeb5 100644 --- a/tests/unit/server_tests/test_service_info_handler.py +++ b/tests/unit/server_tests/test_service_info_handler.py @@ -1,8 +1,8 @@ from argparse import Namespace import json import os -from tabpy_server.app.app import TabPyApp -from tabpy_server.app.SettingsParameters import SettingsParameters +from tabpy.tabpy_server.app.app import TabPyApp +from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters import tempfile from tornado.testing import AsyncHTTPTestCase from unittest.mock import patch @@ -23,7 +23,7 @@ class TestServiceInfoHandlerDefault(AsyncHTTPTestCase): @classmethod def setUpClass(cls): cls.patcher = patch( - 'tabpy_server.app.app.TabPyApp._parse_cli_arguments', + 'tabpy.tabpy_server.app.app.TabPyApp._parse_cli_arguments', return_value=Namespace( config=None)) cls.patcher.start() diff --git a/tests/unit/tools_tests/test_client.py b/tests/unit/tools_tests/test_client.py index 1a7c18dc..670069d0 100644 --- a/tests/unit/tools_tests/test_client.py +++ b/tests/unit/tools_tests/test_client.py @@ -1,7 +1,7 @@ import unittest from unittest.mock import Mock -from tabpy_tools.client import Client +from tabpy.tabpy_tools.client import Client class TestClient(unittest.TestCase): diff --git a/tests/unit/tools_tests/test_rest.py b/tests/unit/tools_tests/test_rest.py index bbd29f1a..530abfee 100644 --- a/tests/unit/tools_tests/test_rest.py +++ b/tests/unit/tools_tests/test_rest.py @@ -1,11 +1,10 @@ import json import requests +from requests.auth import HTTPBasicAuth import sys +from tabpy.tabpy_tools.rest import (RequestsNetworkWrapper, ServiceClient) import unittest from unittest.mock import Mock -from requests.auth import HTTPBasicAuth - -from tabpy_tools.rest import (RequestsNetworkWrapper, ServiceClient) class TestRequestsNetworkWrapper(unittest.TestCase): @@ -20,19 +19,19 @@ def test_init_with_session(self): self.assertIs(session, rnw.session) - def setUp(self): - def mock_response(status_code): - response = Mock(requests.Response()) - response.json.return_value = 'json' - response.status_code = status_code + def mock_response(self, status_code): + response = Mock(requests.Response()) + response.json.return_value = 'json' + response.status_code = status_code - return response + return response + def setUp(self): session = Mock(requests.session()) - session.get.return_value = mock_response(200) - session.post.return_value = mock_response(200) - session.put.return_value = mock_response(200) - session.delete.return_value = mock_response(204) + session.get.return_value = self.mock_response(200) + session.post.return_value = self.mock_response(200) + session.put.return_value = self.mock_response(200) + session.delete.return_value = self.mock_response(204) self.rnw = RequestsNetworkWrapper(session=session) @@ -46,18 +45,18 @@ def test_GET(self): timeout=None, auth=None) - @unittest.expectedFailure def test_GET_InvalidData(self): url = 'abc' data = {'cat'} with self.assertRaises(TypeError): + self.rnw.session.get.return_value = self.mock_response(404) self.rnw.GET(url, data) - @unittest.expectedFailure def test_GET_InvalidURL(self): url = '' data = {'foo': 'bar'} with self.assertRaises(TypeError): + self.rnw.session.get.return_value = self.mock_response(404) self.rnw.GET(url, data) def test_POST(self): @@ -70,11 +69,11 @@ def test_POST(self): timeout=None, auth=None) - @unittest.expectedFailure def test_POST_InvalidURL(self): url = '' data = {'foo': 'bar'} with self.assertRaises(TypeError): + self.rnw.session.post.return_value = self.mock_response(404) self.rnw.POST(url, data) def test_POST_InvalidData(self): diff --git a/tests/unit/tools_tests/test_rest_object.py b/tests/unit/tools_tests/test_rest_object.py index e673d1c6..ec96511e 100644 --- a/tests/unit/tools_tests/test_rest_object.py +++ b/tests/unit/tools_tests/test_rest_object.py @@ -1,7 +1,7 @@ import unittest import sys -from tabpy_tools.rest import RESTObject, RESTProperty, enum +from tabpy.tabpy_tools.rest import RESTObject, RESTProperty, enum class TestRESTObject(unittest.TestCase): diff --git a/tests/unit/tools_tests/test_schema.py b/tests/unit/tools_tests/test_schema.py new file mode 100755 index 00000000..0b7802d2 --- /dev/null +++ b/tests/unit/tools_tests/test_schema.py @@ -0,0 +1,41 @@ +import unittest +import json +from unittest.mock import Mock + +from tabpy.tabpy_tools.schema import generate_schema + + +class TestSchema(unittest.TestCase): + + def test_schema(self): + schema = generate_schema( + input={'x': ['happy', 'sad', 'neutral']}, + input_description={'x': 'text to analyze'}, + output=[.98, -0.99, 0], + output_description='scores for input texts') + expected = { + 'input': { + 'type': 'object', + 'properties': { + 'x': { + 'type': 'array', + 'items': { + 'type': 'string' + }, + 'description': 'text to analyze' + } + }, + 'required': ['x'] + }, + 'sample': { + 'x': ['happy', 'sad', 'neutral'] + }, + 'output': { + 'type': 'array', + 'items': { + 'type': 'number' + }, + 'description': 'scores for input texts' + } + } + self.assertEqual(schema, expected) diff --git a/utils/set_env.cmd b/utils/set_env.cmd deleted file mode 100755 index 45baae54..00000000 --- a/utils/set_env.cmd +++ /dev/null @@ -1,2 +0,0 @@ -@ECHO off -SET PYTHONPATH=%PYTHONPATH%;./tabpy-server;./tabpy-tools; \ No newline at end of file diff --git a/utils/set_env.sh b/utils/set_env.sh deleted file mode 100755 index 4f683fcf..00000000 --- a/utils/set_env.sh +++ /dev/null @@ -1 +0,0 @@ -export PYTHONPATH=./tabpy-server:./tabpy-tools:$PYTHONPATH \ No newline at end of file