From a44798ea89551b5d12c3e769d651ec8e903ec414 Mon Sep 17 00:00:00 2001 From: Jessica Date: Wed, 22 May 2019 11:37:23 -0700 Subject: [PATCH 01/28] Added parameter for configuring timeout for /evaluate. (#298) * Added configurable timeout. * Added unit test for custom evaluate timeout. * Fixed evaluation timeout, added integration tests + additional UT. * Documentation updates. * Reverting state.ini. * Updated file path in server-config.md. * Fixing pep8 issues. * Improved timeout error messaging. * Fixed docs, removed debugging prints. * Returning HTTP code 408 on eval timeout. --- docs/server-config.md | 10 ++++ .../tabpy_server/app/ConfigParameters.py | 1 + .../tabpy_server/app/SettingsParameters.py | 1 + tabpy-server/tabpy_server/app/app.py | 10 ++++ tabpy-server/tabpy_server/common/default.conf | 5 ++ .../tabpy_server/handlers/base_handler.py | 1 + .../handlers/evaluation_plane_handler.py | 27 ++++++--- .../handlers/query_plane_handler.py | 1 - tabpy-server/tabpy_server/tabpy.py | 1 - tests/integration/integ_test_base.py | 20 ++++++- .../test_custom_evaluate_timeout.py | 57 +++++++++++++++++++ tests/unit/server_tests/test_config.py | 44 ++++++++++++-- 12 files changed, 161 insertions(+), 17 deletions(-) create mode 100644 tests/integration/test_custom_evaluate_timeout.py diff --git a/docs/server-config.md b/docs/server-config.md index 4cf188b3..66bec854 100755 --- a/docs/server-config.md +++ b/docs/server-config.md @@ -14,6 +14,7 @@ * [Deleting an Account](#deleting-an-account) - [Logging](#logging) * [Request Context Logging](#request-context-logging) +- [Custom Script Timeout](#custom-script-timeout) @@ -192,3 +193,12 @@ arg1, _arg2): No passwords are logged. NOTE the request context details are logged with INFO level. + +## Custom Script Timeout + +By default, all custom scripts executed through `POST /evaluate` may run for up +to 30.0 s before being terminated. To configure this timeout, uncomment +`TABPY_EVALUATE_TIMEOUT = 30` in the default config under +`tabpy-server/tabpy_server/common/default.conf` and replace `30` with the float +value of your choice representing the timeout time in seconds, or add such an +entry to your custom config. diff --git a/tabpy-server/tabpy_server/app/ConfigParameters.py b/tabpy-server/tabpy_server/app/ConfigParameters.py index 0af2c5af..9ff1a4ad 100644 --- a/tabpy-server/tabpy_server/app/ConfigParameters.py +++ b/tabpy-server/tabpy_server/app/ConfigParameters.py @@ -12,3 +12,4 @@ class ConfigParameters: TABPY_PWD_FILE = 'TABPY_PWD_FILE' TABPY_LOG_DETAILS = 'TABPY_LOG_DETAILS' TABPY_STATIC_PATH = 'TABPY_STATIC_PATH' + TABPY_EVALUATE_TIMEOUT = 'TABPY_EVALUATE_TIMEOUT' diff --git a/tabpy-server/tabpy_server/app/SettingsParameters.py b/tabpy-server/tabpy_server/app/SettingsParameters.py index b070a2b4..562b6de4 100755 --- a/tabpy-server/tabpy_server/app/SettingsParameters.py +++ b/tabpy-server/tabpy_server/app/SettingsParameters.py @@ -12,3 +12,4 @@ class SettingsParameters: ApiVersions = 'versions' LogRequestContext = 'log_request_context' StaticPath = 'static_path' + EvaluateTimeout = 'evaluate_timeout' diff --git a/tabpy-server/tabpy_server/app/app.py b/tabpy-server/tabpy_server/app/app.py index df175304..ec911481 100644 --- a/tabpy-server/tabpy_server/app/app.py +++ b/tabpy-server/tabpy_server/app/app.py @@ -186,6 +186,16 @@ def set_parameter(settings_key, set_parameter(SettingsParameters.ServerVersion, None, default_val=__version__) + set_parameter(SettingsParameters.EvaluateTimeout, ConfigParameters.TABPY_EVALUATE_TIMEOUT, + default_val=30) + try: + self.settings[SettingsParameters.EvaluateTimeout] = float( + self.settings[SettingsParameters.EvaluateTimeout]) + except ValueError: + logger.warning( + 'Evaluate timeout must be a float type. Defaulting to evaluate timeout of 30 seconds.') + self.settings[SettingsParameters.EvaluateTimeout] = 30 + set_parameter(SettingsParameters.UploadDir, ConfigParameters.TABPY_QUERY_OBJECT_PATH, default_val='/tmp/query_objects', check_env_var=True) diff --git a/tabpy-server/tabpy_server/common/default.conf b/tabpy-server/tabpy_server/common/default.conf index 6eef5265..5dd79d53 100755 --- a/tabpy-server/tabpy_server/common/default.conf +++ b/tabpy-server/tabpy_server/common/default.conf @@ -20,6 +20,11 @@ TABPY_STATIC_PATH = ./tabpy-server/tabpy_server/static # end user info if provided. # TABPY_LOG_DETAILS = true +# Configure how long a custom script provided to the /evaluate method +# will run before throwing a TimeoutError. +# The value should be a float representing the timeout time in seconds. +#TABPY_EVALUATE_TIMEOUT = 30 + [loggers] keys=root diff --git a/tabpy-server/tabpy_server/handlers/base_handler.py b/tabpy-server/tabpy_server/handlers/base_handler.py index 3407ba76..80fb6ad8 100644 --- a/tabpy-server/tabpy_server/handlers/base_handler.py +++ b/tabpy-server/tabpy_server/handlers/base_handler.py @@ -130,6 +130,7 @@ def initialize(self, app): self.credentials = app.credentials self.username = None self.password = None + self.eval_timeout = self.settings[SettingsParameters.EvaluateTimeout] self.logger = ContextLoggerWrapper(self.request) self.logger.enable_context_logging( diff --git a/tabpy-server/tabpy_server/handlers/evaluation_plane_handler.py b/tabpy-server/tabpy_server/handlers/evaluation_plane_handler.py index f32cee67..42ff647d 100644 --- a/tabpy-server/tabpy_server/handlers/evaluation_plane_handler.py +++ b/tabpy-server/tabpy_server/handlers/evaluation_plane_handler.py @@ -5,13 +5,14 @@ import logging from tabpy_server.common.util import format_exception import requests -import sys +from concurrent.futures import TimeoutError class RestrictedTabPy: - def __init__(self, port, logger): + def __init__(self, port, logger, timeout): self.port = port self.logger = logger + self.timeout = timeout def query(self, name, *args, **kwargs): url = f'http://localhost:{self.port}/query/{name}' @@ -20,8 +21,7 @@ def query(self, name, *args, **kwargs): data = json.dumps(internal_data) headers = {'content-type': 'application/json'} response = requests.post(url=url, data=data, headers=headers, - timeout=30) - + timeout=self.timeout) return response.json() @@ -33,6 +33,7 @@ class EvaluationPlaneHandler(BaseHandler): def initialize(self, executor, app): super(EvaluationPlaneHandler, self).initialize(app) self.executor = executor + self._error_message_timeout = f'User defined script timed out. Timeout is set to {self.eval_timeout} s.' @tornado.web.asynchronous @gen.coroutine @@ -79,8 +80,12 @@ def post(self): logging.INFO, f'function to evaluate={function_to_evaluate}') - result = yield self.call_subprocess(function_to_evaluate, - arguments) + try: + result = yield self.call_subprocess(function_to_evaluate, arguments) + except TimeoutError: + self.error_out(408, self._error_message_timeout) + return + if result is None: self.error_out(400, 'Error running script. No return value') else: @@ -103,7 +108,8 @@ def post(self): @gen.coroutine def call_subprocess(self, function_to_evaluate, arguments): - restricted_tabpy = RestrictedTabPy(self.port, self) + restricted_tabpy = RestrictedTabPy( + self.port, self.logger, self.eval_timeout) # Exec does not run the function, so it does not block. exec(function_to_evaluate, globals()) @@ -112,5 +118,10 @@ def call_subprocess(self, function_to_evaluate, arguments): else: future = self.executor.submit(_user_script, restricted_tabpy, **arguments) - ret = yield future + + try: + ret = future.result(timeout=self.eval_timeout) + except TimeoutError: + self.logger.log(logging.ERROR, self._error_message_timeout) + raise TimeoutError raise gen.Return(ret) diff --git a/tabpy-server/tabpy_server/handlers/query_plane_handler.py b/tabpy-server/tabpy_server/handlers/query_plane_handler.py index 230d1727..b1b03182 100644 --- a/tabpy-server/tabpy_server/handlers/query_plane_handler.py +++ b/tabpy-server/tabpy_server/handlers/query_plane_handler.py @@ -8,7 +8,6 @@ import json from tabpy_server.common.util import format_exception import urllib -import sys import tornado.web diff --git a/tabpy-server/tabpy_server/tabpy.py b/tabpy-server/tabpy_server/tabpy.py index f7936c5a..7df99751 100644 --- a/tabpy-server/tabpy_server/tabpy.py +++ b/tabpy-server/tabpy_server/tabpy.py @@ -1,4 +1,3 @@ -from tabpy_server import __version__ from tabpy_server.app.app import TabPyApp diff --git a/tests/integration/integ_test_base.py b/tests/integration/integ_test_base.py index 331a0f11..87d11d55 100755 --- a/tests/integration/integ_test_base.py +++ b/tests/integration/integ_test_base.py @@ -13,6 +13,7 @@ class IntegTestBase(unittest.TestCase): ''' Base class for integration tests. ''' + def __init__(self, methodName="runTest"): super(IntegTestBase, self).__init__(methodName) self.process = None @@ -137,6 +138,19 @@ def _get_key_file_name(self) -> str: ''' return None + def _get_evaluate_timeout(self) -> str: + ''' + Returns the configured timeout for the /evaluate method. + Default implementation returns None, which means that the timeout will default to 30. + + Returns + ------- + str + Timeout for calling /evaluate. + If None, defaults TABPY_EVALUATE_TIMEOUT setting will default to '30'. + ''' + return None + def _get_config_file_name(self) -> str: ''' Generates config file. Overwrite this function for tests to @@ -173,6 +187,10 @@ def _get_config_file_name(self) -> str: key_file_name = os.path.abspath(key_file_name) config_file.write(f'TABPY_KEY_FILE = {key_file_name}\n') + evaluate_timeout = self._get_evaluate_timeout() + if evaluate_timeout is not None: + config_file.write(f'TABPY_EVALUATE_TIMEOUT = {evaluate_timeout}\n') + config_file.close() self.delete_config_file = True @@ -224,7 +242,7 @@ def tearDown(self): if self.process is not None: if platform.system() == 'Windows': subprocess.call(['taskkill', '/F', '/T', '/PID', - str(self.process.pid)]) + str(self.process.pid)]) else: os.killpg(os.getpgid(self.process.pid), signal.SIGTERM) self.process.kill() diff --git a/tests/integration/test_custom_evaluate_timeout.py b/tests/integration/test_custom_evaluate_timeout.py new file mode 100644 index 00000000..303bea58 --- /dev/null +++ b/tests/integration/test_custom_evaluate_timeout.py @@ -0,0 +1,57 @@ +import integ_test_base +from tabpy_tools.client import Client + + +class TestCustomEvaluateTimeout(integ_test_base.IntegTestBase): + def __init__(self, *args, **kwargs): + super(TestCustomEvaluateTimeout, self).__init__(*args, **kwargs) + self._headers = { + 'Content-Type': "application/json", + 'TabPy-Client': "Integration test for testing custom evaluate timeouts." + } + self._expected_error_message = '{"message": "User defined script timed out. Timeout is set to 5.0 s.", "info": {}}' + + def _get_evaluate_timeout(self) -> str: + return '5' + + def test_custom_evaluate_timeout_with_script(self): + payload = ( + ''' + { + "data": { "_arg1": 1 }, + "script": "import time\\nwhile True:\\n time.sleep(1)\\nreturn 1" + } + ''') + + self._run_test(payload) + + def test_custom_evaluate_timeout_with_model(self): + # deploy spin + def spin(): + import time + while True: + time.sleep(1) + return 1 + + client = Client(f'http://localhost:{self._get_port()}/') + client.deploy('spin', spin, 'Spins indefinitely for testing purposes.') + + payload = ( + ''' + { + "data": {"_arg1": 1}, + "script": "return tabpy.query('spin')" + + } + ''') + + self._run_test(payload) + + def _run_test(self, payload): + conn = self._get_connection() + conn.request('POST', '/evaluate', payload, self._headers) + res = conn.getresponse() + actual_error_message = res.read().decode('utf-8') + + self.assertEqual(408, res.status) + self.assertEqual(self._expected_error_message, actual_error_message) diff --git a/tests/unit/server_tests/test_config.py b/tests/unit/server_tests/test_config.py index 58a67b53..38b7acf9 100644 --- a/tests/unit/server_tests/test_config.py +++ b/tests/unit/server_tests/test_config.py @@ -62,6 +62,13 @@ def test_no_state_ini_file_or_state_dir(self, mock_os, mock_file_exists, class TestPartialConfigFile(unittest.TestCase): + def setUp(self): + self.config_file = NamedTemporaryFile(delete=False) + + def tearDown(self): + os.remove(self.config_file.name) + self.config_file = None + @patch('tabpy_server.app.app.TabPyApp._parse_cli_arguments') @patch('tabpy_server.app.app.TabPyState') @patch('tabpy_server.app.app._get_state_from_file') @@ -71,11 +78,11 @@ class TestPartialConfigFile(unittest.TestCase): def test_config_file_present(self, mock_os, mock_path_exists, mock_psws, mock_management_util, mock_tabpy_state, mock_parse_arguments): - config_file = NamedTemporaryFile(delete=False) - - config_file.write("[TabPy]\n" - "TABPY_QUERY_OBJECT_PATH = foo\n" - "TABPY_STATE_PATH = bar\n".encode()) + self.assertTrue(self.config_file is not None) + config_file = self.config_file + config_file.write('[TabPy]\n' + 'TABPY_QUERY_OBJECT_PATH = foo\n' + 'TABPY_STATE_PATH = bar\n'.encode()) config_file.close() mock_parse_arguments.return_value = Namespace(config=config_file.name) @@ -96,8 +103,33 @@ def test_config_file_present(self, mock_os, mock_path_exists, self.assertTrue('certificate_file' not in app.settings) self.assertTrue('key_file' not in app.settings) self.assertEqual(app.settings['log_request_context'], False) + self.assertEqual(app.settings['evaluate_timeout'], 30) + + @patch('tabpy_server.app.app.os.path.exists', return_value=True) + @patch('tabpy_server.app.app._get_state_from_file') + @patch('tabpy_server.app.app.TabPyState') + def test_custom_evaluate_timeout_valid(self, mock_state, mock_get_state_from_file, mock_path_exists): + self.assertTrue(self.config_file is not None) + config_file = self.config_file + config_file.write('[TabPy]\n' + 'TABPY_EVALUATE_TIMEOUT = 1996'.encode()) + config_file.close() + + app = TabPyApp(self.config_file.name) + self.assertEqual(app.settings['evaluate_timeout'], 1996.0) + + @patch('tabpy_server.app.app.os.path.exists', return_value=True) + @patch('tabpy_server.app.app._get_state_from_file') + @patch('tabpy_server.app.app.TabPyState') + def test_custom_evaluate_timeout_invalid(self, mock_state, mock_get_state_from_file, mock_path_exists): + self.assertTrue(self.config_file is not None) + config_file = self.config_file + config_file.write('[TabPy]\n' + 'TABPY_EVALUATE_TIMEOUT = "im not a float"'.encode()) + config_file.close() - os.remove(config_file.name) + app = TabPyApp(self.config_file.name) + self.assertEqual(app.settings['evaluate_timeout'], 30.0) class TestTransferProtocolValidation(unittest.TestCase): From b68e5c1292d0ebc3ad3a0836d8c677e61925f461 Mon Sep 17 00:00:00 2001 From: Jessica Date: Wed, 29 May 2019 13:55:44 -0700 Subject: [PATCH 02/28] Querying models through /evaluate causes server hang. (#300) * Made call_subprocess non-blocking. * Magic? * Using native coroutines in EvaluationPlaneHandler. * Updated docs to reflect timeout limitations. * Resolved markdownlint issue. * Back to tornado coroutines. * Fixed pep8 style. --- docs/server-config.md | 3 ++ .../handlers/evaluation_plane_handler.py | 26 +++++----- .../test_custom_evaluate_timeout.py | 52 ++++++------------- 3 files changed, 31 insertions(+), 50 deletions(-) diff --git a/docs/server-config.md b/docs/server-config.md index 66bec854..a57bd363 100755 --- a/docs/server-config.md +++ b/docs/server-config.md @@ -202,3 +202,6 @@ to 30.0 s before being terminated. To configure this timeout, uncomment `tabpy-server/tabpy_server/common/default.conf` and replace `30` with the float value of your choice representing the timeout time in seconds, or add such an entry to your custom config. + +This timeout does not apply when evaluating models either through the `/query` +method, or using the `tabpy.query(...)` syntax with the `/evaluate` method. diff --git a/tabpy-server/tabpy_server/handlers/evaluation_plane_handler.py b/tabpy-server/tabpy_server/handlers/evaluation_plane_handler.py index 42ff647d..8b61f0b1 100644 --- a/tabpy-server/tabpy_server/handlers/evaluation_plane_handler.py +++ b/tabpy-server/tabpy_server/handlers/evaluation_plane_handler.py @@ -1,11 +1,10 @@ from tabpy_server.handlers import BaseHandler -import tornado.web -from tornado import gen import json import logging from tabpy_server.common.util import format_exception import requests -from concurrent.futures import TimeoutError +from tornado import gen +from datetime import timedelta class RestrictedTabPy: @@ -33,9 +32,9 @@ class EvaluationPlaneHandler(BaseHandler): def initialize(self, executor, app): super(EvaluationPlaneHandler, self).initialize(app) self.executor = executor - self._error_message_timeout = f'User defined script timed out. Timeout is set to {self.eval_timeout} s.' + self._error_message_timeout = f'User defined script timed out. ' \ + f'Timeout is set to {self.eval_timeout} s.' - @tornado.web.asynchronous @gen.coroutine def post(self): if self.should_fail_with_not_authorized(): @@ -81,8 +80,12 @@ def post(self): f'function to evaluate={function_to_evaluate}') try: - result = yield self.call_subprocess(function_to_evaluate, arguments) - except TimeoutError: + result = yield self._call_subprocess(function_to_evaluate, + arguments) + except (gen.TimeoutError, + requests.exceptions.ConnectTimeout, + requests.exceptions.ReadTimeout): + self.logger.log(logging.ERROR, self._error_message_timeout) self.error_out(408, self._error_message_timeout) return @@ -107,7 +110,7 @@ def post(self): "provided.") @gen.coroutine - def call_subprocess(self, function_to_evaluate, arguments): + def _call_subprocess(self, function_to_evaluate, arguments): restricted_tabpy = RestrictedTabPy( self.port, self.logger, self.eval_timeout) # Exec does not run the function, so it does not block. @@ -119,9 +122,6 @@ def call_subprocess(self, function_to_evaluate, arguments): future = self.executor.submit(_user_script, restricted_tabpy, **arguments) - try: - ret = future.result(timeout=self.eval_timeout) - except TimeoutError: - self.logger.log(logging.ERROR, self._error_message_timeout) - raise TimeoutError + ret = yield gen.with_timeout(timedelta(seconds=self.eval_timeout), + future) raise gen.Return(ret) diff --git a/tests/integration/test_custom_evaluate_timeout.py b/tests/integration/test_custom_evaluate_timeout.py index 303bea58..dafb0f3b 100644 --- a/tests/integration/test_custom_evaluate_timeout.py +++ b/tests/integration/test_custom_evaluate_timeout.py @@ -1,16 +1,7 @@ import integ_test_base -from tabpy_tools.client import Client class TestCustomEvaluateTimeout(integ_test_base.IntegTestBase): - def __init__(self, *args, **kwargs): - super(TestCustomEvaluateTimeout, self).__init__(*args, **kwargs) - self._headers = { - 'Content-Type': "application/json", - 'TabPy-Client': "Integration test for testing custom evaluate timeouts." - } - self._expected_error_message = '{"message": "User defined script timed out. Timeout is set to 5.0 s.", "info": {}}' - def _get_evaluate_timeout(self) -> str: return '5' @@ -19,39 +10,26 @@ def test_custom_evaluate_timeout_with_script(self): ''' { "data": { "_arg1": 1 }, - "script": "import time\\nwhile True:\\n time.sleep(1)\\nreturn 1" + "script": + "import time\\nwhile True:\\n time.sleep(1)\\nreturn 1" } ''') + headers = { + 'Content-Type': + "application/json", + 'TabPy-Client': + "Integration test for testing custom evaluate timeouts with " + "scripts." + } - self._run_test(payload) - - def test_custom_evaluate_timeout_with_model(self): - # deploy spin - def spin(): - import time - while True: - time.sleep(1) - return 1 - - client = Client(f'http://localhost:{self._get_port()}/') - client.deploy('spin', spin, 'Spins indefinitely for testing purposes.') - - payload = ( - ''' - { - "data": {"_arg1": 1}, - "script": "return tabpy.query('spin')" - - } - ''') - - self._run_test(payload) - - def _run_test(self, payload): conn = self._get_connection() - conn.request('POST', '/evaluate', payload, self._headers) + conn.request('POST', '/evaluate', payload, headers) res = conn.getresponse() actual_error_message = res.read().decode('utf-8') + self.assertEqual( + '{"message": ' + '"User defined script timed out. Timeout is set to 5.0 s.", ' + '"info": {}}', + actual_error_message) self.assertEqual(408, res.status) - self.assertEqual(self._expected_error_message, actual_error_message) From fb581bfbb10a6878099a86afc377dae2349decbb Mon Sep 17 00:00:00 2001 From: sbabayan <34922408+sbabayan@users.noreply.github.com> Date: Thu, 30 May 2019 10:02:50 -0700 Subject: [PATCH 03/28] fixing regex issue where . were not allowed in endpoint path (#305) (#306) --- tabpy-server/tabpy_server/handlers/management_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tabpy-server/tabpy_server/handlers/management_handler.py b/tabpy-server/tabpy_server/handlers/management_handler.py index 83c4c599..972876f3 100644 --- a/tabpy-server/tabpy_server/handlers/management_handler.py +++ b/tabpy-server/tabpy_server/handlers/management_handler.py @@ -94,7 +94,7 @@ def _add_or_update_endpoint(self, action, name, version, request_data): self.settings[SettingsParameters.StateFilePath], name, version) self.logger.log(logging.DEBUG, f'Checking source path {src_path}...') - _path_checker = _compile(r'^[\\\:a-zA-Z0-9-_~\s/]+$') + _path_checker = _compile(r'^[\\\:a-zA-Z0-9-_~\s/\.]+$') # copy from staging if src_path: if not isinstance(request_data['src_path'], str): From 65ed537c24e0839070aa5abea1077865be446cda Mon Sep 17 00:00:00 2001 From: sbabayan <34922408+sbabayan@users.noreply.github.com> Date: Fri, 7 Jun 2019 09:41:53 -0700 Subject: [PATCH 04/28] adding t-test to pre-deployed models (#312) * adding t-test to pre-deployed models * fixed pep8 issues * import error on ttest and updated successful deployment message for sentiment analysis * fixed md issues * fixed endpoint name in integration tests * refactor redundant code and update md --- docs/tabpy-tools.md | 45 +++++++++++++++++++ models/scripts/PCA.py | 26 ++--------- models/scripts/SentimentAnalysis.py | 27 ++--------- models/scripts/tTest.py | 44 ++++++++++++++++++ models/utils/setup_utils.py | 23 ++++++++++ .../test_deploy_model_ssl_off_auth_off.py | 15 +++---- .../test_deploy_model_ssl_off_auth_on.py | 17 +++---- .../test_deploy_model_ssl_on_auth_off.py | 14 +++--- .../test_deploy_model_ssl_on_auth_on.py | 16 +++---- 9 files changed, 143 insertions(+), 84 deletions(-) create mode 100644 models/scripts/tTest.py diff --git a/docs/tabpy-tools.md b/docs/tabpy-tools.md index 5184f5e3..d26bc1a7 100755 --- a/docs/tabpy-tools.md +++ b/docs/tabpy-tools.md @@ -8,6 +8,7 @@ on TabPy server. - [Connecting to TabPy](#connecting-to-tabpy) - [Authentication](#authentication) - [Deploying a Function](#deploying-a-function) +- [Predeployed Functions](#predeployed-functions) - [Providing Schema Metadata](#providing-schema-metadata) - [Querying an Endpoint](#querying-an-endpoint) - [Evaluating Arbitrary Python Scripts](#evaluating-arbitrary-python-scripts) @@ -265,6 +266,50 @@ tabpy.query('Sentiment Analysis', _arg1, library='textblob')[‘response’] ``` +### T-Test + +A [t-test](https://en.wikipedia.org/wiki/Student%27s_t-test) is a statistical +hypothesis test that is used to compare two sample means or a sample’s mean against +a known population mean. The ttest should be used when the means of the samples +follows a normal distribution but the variance may not be known. + +TabPy’s pre-deployed t-test implementation can be called using the following syntax, + +```python + +tabpy.query(‘ttest’, _arg1, _arg2)[‘response’] + +``` + +and is capable of performing two types of t-tests: + + +1\. [A t-test for the means of two independent samples with equal variance](https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.ttest_ind.html) +This is a two-sided t test with the null hypothesis being that the mean of +sample1 is equal to the mean of sample2. +_arg1 (list of numeric values): a list of independent observations +_arg2 (list of numeric values): a list of independent observations equal to +the length of _arg1 + +Alternatively, your data may not be split into separate measures. If that is +the case you can pass the following fields to ttest, + +_arg1 (list of numeric values): a list of independent observations +_arg2 (list of categorical variables with cardinality two): a binary factor +that maps each observation in _arg1 to either sample1 or sample2 (this list +should be equal to the length of _arg1) + +2\. [A t-test for the mean of one group](https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.stats.ttest_1samp.html) +_arg1 (list of numeric values): a list of independent observations +_arg2 (a numeric value): the known population mean +A two-sided t test with the null hypothesis being that the mean of a sample of +independent observations is equal to the given population mean. + +The function returns a two-tailed [p-value](https://en.wikipedia.org/wiki/P-value) +(between 0 and 1). Depending on your [significance level](https://en.wikipedia.org/wiki/Statistical_significance) +you may reject or fail to reject the null hypothesis. + + ## Providing Schema Metadata As soon as you share your deployed functions, you also need to share metadata diff --git a/models/scripts/PCA.py b/models/scripts/PCA.py index e54b2046..9c2d68b2 100644 --- a/models/scripts/PCA.py +++ b/models/scripts/PCA.py @@ -1,4 +1,3 @@ -from tabpy_tools.client import Client import pandas as pd from numpy import array from sklearn.decomposition import PCA as sklearnPCA @@ -60,25 +59,6 @@ def PCA(component, _arg1, _arg2, *_argN): if __name__ == '__main__': - # running from setup.py - if len(sys.argv) > 1: - config_file_path = sys.argv[1] - else: - config_file_path = setup_utils.get_default_config_file_path() - port, auth_on, prefix = setup_utils.parse_config(config_file_path) - - connection = Client(f'{prefix}://localhost:{port}/') - - if auth_on: - # credentials are passed in from setup.py - if len(sys.argv) == 4: - user, passwd = sys.argv[2], sys.argv[3] - # running PCA independently - else: - user, passwd = setup_utils.get_creds() - connection.set_credentials(user, passwd) - - connection.deploy('PCA', PCA, - 'Returns the specified principal component.', - override=True) - print("Successfully deployed PCA") + setup_utils.main('PCA', + PCA, + 'Returns the specified principal component') diff --git a/models/scripts/SentimentAnalysis.py b/models/scripts/SentimentAnalysis.py index 55c9416a..0b3c3ab6 100644 --- a/models/scripts/SentimentAnalysis.py +++ b/models/scripts/SentimentAnalysis.py @@ -1,4 +1,3 @@ -from tabpy_tools.client import Client from textblob import TextBlob import nltk from nltk.sentiment.vader import SentimentIntensityAnalyzer @@ -43,25 +42,7 @@ def SentimentAnalysis(_arg1, library='nltk'): if __name__ == '__main__': - # running from setup.py - if len(sys.argv) > 1: - config_file_path = sys.argv[1] - else: - config_file_path = setup_utils.get_default_config_file_path() - port, auth_on, prefix = setup_utils.parse_config(config_file_path) - - connection = Client(f'{prefix}://localhost:{port}/') - - if auth_on: - # credentials are passed in from setup.py - if len(sys.argv) == 4: - user, passwd = sys.argv[2], sys.argv[3] - # running Sentiment Analysis independently - else: - user, passwd = setup_utils.get_creds() - connection.set_credentials(user, passwd) - - connection.deploy('Sentiment Analysis', SentimentAnalysis, - 'Returns a sentiment score between -1 and ' - '1 for a given string.', override=True) - print("Successfully deployed SentimentAnalysis") + setup_utils.main('Sentiment Analysis', + SentimentAnalysis, + 'Returns a sentiment score between -1 and 1 for ' + 'a given string') diff --git a/models/scripts/tTest.py b/models/scripts/tTest.py new file mode 100644 index 00000000..d7082698 --- /dev/null +++ b/models/scripts/tTest.py @@ -0,0 +1,44 @@ +from scipy import stats +import sys +from pathlib import Path +sys.path.append(str(Path(__file__).resolve().parent.parent.parent / 'models')) +from utils import setup_utils + + +def ttest(_arg1, _arg2): + ''' + T-Test is a statistical hypothesis test that is used to compare + two sample means or a sample’s mean against a known population mean. + For more information on the function and how to use it please refer + to tabpy-tools.md + ''' + # one sample test with mean + if len(_arg2) == 1: + test_stat, p_value = stats.ttest_1samp(_arg1, _arg2) + return p_value + # two sample t-test where _arg1 is numeric and _arg2 is a binary factor + elif len(set(_arg2)) == 2: + # each sample in _arg1 needs to have a corresponding classification + # in _arg2 + if not (len(_arg1) == len(_arg2)): + raise ValueError + class1, class2 = set(_arg2) + sample1 = [] + sample2 = [] + for i in range(len(_arg1)): + if _arg2[i] == class1: + sample1.append(_arg1[i]) + else: + sample2.append(_arg1[i]) + test_stat, p_value = stats.ttest_ind(sample1, sample2, equal_var=False) + return p_value + # arg1 is a sample and arg2 is a sample + else: + test_stat, p_value = stats.ttest_ind(_arg1, _arg2, equal_var=False) + return p_value + + +if __name__ == '__main__': + setup_utils.main('ttest', + ttest, + 'Returns the p-value form a t-test') diff --git a/models/utils/setup_utils.py b/models/utils/setup_utils.py index de8416e7..e3da48de 100644 --- a/models/utils/setup_utils.py +++ b/models/utils/setup_utils.py @@ -2,6 +2,7 @@ from pathlib import Path import getpass import sys +from tabpy_tools.client import Client def get_default_config_file_path(): @@ -31,3 +32,25 @@ def get_creds(): user = sys.stdin.readline().rstrip() passwd = sys.stdin.readline().rstrip() return [user, passwd] + +def main(funcName, func, funcDescription): + # running from setup.py + if len(sys.argv) > 1: + config_file_path = sys.argv[1] + else: + config_file_path = get_default_config_file_path() + port, auth_on, prefix = parse_config(config_file_path) + + connection = Client(f'{prefix}://localhost:{port}/') + + if auth_on: + # credentials are passed in from setup.py + if len(sys.argv) == 4: + user, passwd = sys.argv[2], sys.argv[3] + # running Sentiment Analysis independently + else: + user, passwd = get_creds() + connection.set_credentials(user, passwd) + + connection.deploy(funcName, func, funcDescription, override=True) + print(f'Successfully deployed {funcName}') diff --git a/tests/integration/test_deploy_model_ssl_off_auth_off.py b/tests/integration/test_deploy_model_ssl_off_auth_off.py index e9059569..e35d81d8 100644 --- a/tests/integration/test_deploy_model_ssl_off_auth_off.py +++ b/tests/integration/test_deploy_model_ssl_off_auth_off.py @@ -5,16 +5,13 @@ class TestDeployModelSSLOffAuthOff(integ_test_base.IntegTestBase): def test_deploy_ssl_off_auth_off(self): + models = ['PCA', 'Sentiment%20Analysis', "ttest"] path = str(Path('models', 'setup.py')) subprocess.call([self.py, path, self._get_config_file_name()]) conn = self._get_connection() - conn.request("GET", "/endpoints/PCA") - PCA_request = conn.getresponse() - self.assertEqual(200, PCA_request.status) - PCA_request.read() - - conn.request("GET", "/endpoints/Sentiment%20Analysis") - SentimentAnalysis_request = conn.getresponse() - self.assertEqual(200, SentimentAnalysis_request.status) - SentimentAnalysis_request.read() + for m in models: + conn.request("GET", f'/endpoints/{m}') + m_request = conn.getresponse() + self.assertEqual(200, m_request.status) + m_request.read() diff --git a/tests/integration/test_deploy_model_ssl_off_auth_on.py b/tests/integration/test_deploy_model_ssl_off_auth_on.py index cbdcec6f..bb1268eb 100644 --- a/tests/integration/test_deploy_model_ssl_off_auth_on.py +++ b/tests/integration/test_deploy_model_ssl_off_auth_on.py @@ -9,6 +9,7 @@ def _get_pwd_file(self) -> str: return './tests/integration/resources/pwdfile.txt' def test_deploy_ssl_off_auth_on(self): + models = ['PCA', 'Sentiment%20Analysis', "ttest"] path = str(Path('models', 'setup.py')) p = subprocess.run([self.py, path, self._get_config_file_name()], input=b'user1\nP@ssw0rd\n') @@ -20,15 +21,11 @@ def test_deploy_ssl_off_auth_on(self): 'Basic ' + base64.b64encode('user1:P@ssw0rd'. encode('utf-8')).decode('utf-8') - } + } conn = self._get_connection() - conn.request("GET", "/endpoints/PCA", headers=headers) - PCA_request = conn.getresponse() - self.assertEqual(200, PCA_request.status) - PCA_request.read() - - conn.request("GET", "/endpoints/Sentiment%20Analysis", headers=headers) - SentimentAnalysis_request = conn.getresponse() - self.assertEqual(200, SentimentAnalysis_request.status) - SentimentAnalysis_request.read() + for m in models: + conn.request("GET", f'/endpoints/{m}', headers=headers) + m_request = conn.getresponse() + self.assertEqual(200, m_request.status) + m_request.read() diff --git a/tests/integration/test_deploy_model_ssl_on_auth_off.py b/tests/integration/test_deploy_model_ssl_on_auth_off.py index 65041b16..fe083849 100644 --- a/tests/integration/test_deploy_model_ssl_on_auth_off.py +++ b/tests/integration/test_deploy_model_ssl_on_auth_off.py @@ -15,6 +15,7 @@ def _get_key_file_name(self) -> str: return './tests/integration/resources/2019_04_24_to_3018_08_25.key' def test_deploy_ssl_on_auth_off(self): + models = ['PCA', 'Sentiment%20Analysis', "ttest"] path = str(Path('models', 'setup.py')) subprocess.call([self.py, path, self._get_config_file_name()]) @@ -24,12 +25,7 @@ def test_deploy_ssl_on_auth_off(self): # Do not warn about insecure request requests.packages.urllib3.disable_warnings() - PCA_response = session.get(url=f'{self._get_transfer_protocol()}://' - 'localhost:9004/endpoints/PCA') - self.assertEqual(200, PCA_response.status_code) - - SentimentAnalysis_response = session.get( - url=f'{self._get_transfer_protocol()}://' - 'localhost:9004/endpoints/' - 'Sentiment Analysis') - self.assertEqual(200, SentimentAnalysis_response.status_code) + for m in models: + m_response = session.get(url=f'{self._get_transfer_protocol()}://' + f'localhost:9004/endpoints/{m}') + self.assertEqual(200, m_response.status_code) diff --git a/tests/integration/test_deploy_model_ssl_on_auth_on.py b/tests/integration/test_deploy_model_ssl_on_auth_on.py index 081ace56..742abceb 100644 --- a/tests/integration/test_deploy_model_ssl_on_auth_on.py +++ b/tests/integration/test_deploy_model_ssl_on_auth_on.py @@ -19,6 +19,7 @@ def _get_pwd_file(self) -> str: return './tests/integration/resources/pwdfile.txt' def test_deploy_ssl_on_auth_on(self): + models = ['PCA', 'Sentiment%20Analysis', "ttest"] path = str(Path('models', 'setup.py')) p = subprocess.run([self.py, path, self._get_config_file_name()], input=b'user1\nP@ssw0rd\n') @@ -36,13 +37,8 @@ def test_deploy_ssl_on_auth_on(self): # Do not warn about insecure request requests.packages.urllib3.disable_warnings() - PCA_response = session.get(url=f'{self._get_transfer_protocol()}' - '://localhost:9004/endpoints/PCA', - headers=headers) - self.assertEqual(200, PCA_response.status_code) - - SentimentAnalysis_response = session.get( - url=f'{self._get_transfer_protocol()}' - '://localhost:9004/endpoints/' - 'Sentiment Analysis', headers=headers) - self.assertEqual(200, SentimentAnalysis_response.status_code) + for m in models: + m_response = session.get(url=f'{self._get_transfer_protocol()}://' + f'localhost:9004/endpoints/{m}', + headers=headers) + self.assertEqual(200, m_response.status_code) From 0881a6369597dc2416ae23b8217a5064cba4cbb0 Mon Sep 17 00:00:00 2001 From: Oleksandr Golovatyi Date: Fri, 19 Jul 2019 15:53:14 -0700 Subject: [PATCH 05/28] Add FAQ.md (#325) * Update README.md * Create FAQ.md --- README.md | 3 +++ docs/FAQ.md | 14 ++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 docs/FAQ.md diff --git a/README.md b/README.md index 274cc0a4..5e169ae3 100755 --- a/README.md +++ b/README.md @@ -25,6 +25,9 @@ order: * [Authoring Python calculations in Tableau](docs/TableauConfiguration.md). * [TabPy Tools](docs/tabpy-tools.md) +Troubleshooting: +* [FAQ for configuration, startup and other issues](docs/FAQ.md) + More technical topics: * [Contributing Guide](CONTRIBUTING.md) for TabPy developers diff --git a/docs/FAQ.md b/docs/FAQ.md new file mode 100644 index 00000000..4bd7b029 --- /dev/null +++ b/docs/FAQ.md @@ -0,0 +1,14 @@ +# TabPy Frequently Asked Questions + + + + + + + + +## Startup Issues + +### AttributeError: module 'tornado.web' has no attribute 'asynchronous' + +TabPy uses Tornado 5.1.1. To it to your Python environment run `pip install tornado==5.1.1` and then try to start TabPy again. From 843b839f7837a9916d48f5af5dc04a0bcb9af5c9 Mon Sep 17 00:00:00 2001 From: Oleksandr Golovatyi Date: Fri, 19 Jul 2019 15:59:53 -0700 Subject: [PATCH 06/28] Fix models call via /evaluate in HTTPS (#322) * Fix models call via /evaluate in HTTPS * Restore state.ini --- .vscode/settings.json | 8 ++-- tabpy-server/tabpy_server/app/app.py | 22 +++++++++- .../tabpy_server/handlers/base_handler.py | 1 + .../handlers/evaluation_plane_handler.py | 13 ++++-- tests/integration/integ_test_base.py | 6 ++- .../test_deploy_and_evaluate_model_ssl.py | 41 +++++++++++++++++++ tests/unit/server_tests/test_config.py | 8 +++- 7 files changed, 85 insertions(+), 14 deletions(-) create mode 100755 tests/integration/test_deploy_and_evaluate_model_ssl.py diff --git a/.vscode/settings.json b/.vscode/settings.json index 4ec4389e..4a047b81 100755 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -12,11 +12,11 @@ "python.linting.flake8Enabled": false, "python.linting.enabled": true, "python.testing.autoTestDiscoverOnSaveEnabled": true, - "python.testing.pyTestArgs": [ - "tests" + "python.testing.pytestArgs": [ + "." ], - "python.testing.unittestEnabled": true, + "python.testing.unittestEnabled": false, "python.testing.nosetestsEnabled": false, - "python.testing.pyTestEnabled": true, + "python.testing.pytestEnabled": true, "python.linting.pep8Enabled": true } \ No newline at end of file diff --git a/tabpy-server/tabpy_server/app/app.py b/tabpy-server/tabpy_server/app/app.py index ec911481..0cdc53aa 100644 --- a/tabpy-server/tabpy_server/app/app.py +++ b/tabpy-server/tabpy_server/app/app.py @@ -173,27 +173,45 @@ def set_parameter(settings_key, default_val=None, check_env_var=False): if config_key is not None and\ + parser.has_section('TabPy') and\ parser.has_option('TabPy', config_key): self.settings[settings_key] = parser.get('TabPy', config_key) + logger.debug( + f'Parameter {settings_key} set to ' + f'"{self.settings[settings_key]}" ' + 'from config file') elif check_env_var: self.settings[settings_key] = os.getenv( config_key, default_val) + logger.debug( + f'Parameter {settings_key} set to ' + f'"{self.settings[settings_key]}" ' + 'from environment variable') elif default_val is not None: self.settings[settings_key] = default_val + logger.debug( + f'Parameter {settings_key} set to ' + f'"{self.settings[settings_key]}" ' + 'from default value') + else: + logger.debug( + f'Parameter {settings_key} is not set') set_parameter(SettingsParameters.Port, ConfigParameters.TABPY_PORT, default_val=9004, check_env_var=True) set_parameter(SettingsParameters.ServerVersion, None, default_val=__version__) - set_parameter(SettingsParameters.EvaluateTimeout, ConfigParameters.TABPY_EVALUATE_TIMEOUT, + set_parameter(SettingsParameters.EvaluateTimeout, + ConfigParameters.TABPY_EVALUATE_TIMEOUT, default_val=30) try: self.settings[SettingsParameters.EvaluateTimeout] = float( self.settings[SettingsParameters.EvaluateTimeout]) except ValueError: logger.warning( - 'Evaluate timeout must be a float type. Defaulting to evaluate timeout of 30 seconds.') + 'Evaluate timeout must be a float type. Defaulting ' + 'to evaluate timeout of 30 seconds.') self.settings[SettingsParameters.EvaluateTimeout] = 30 set_parameter(SettingsParameters.UploadDir, diff --git a/tabpy-server/tabpy_server/handlers/base_handler.py b/tabpy-server/tabpy_server/handlers/base_handler.py index 80fb6ad8..3b22f7dc 100644 --- a/tabpy-server/tabpy_server/handlers/base_handler.py +++ b/tabpy-server/tabpy_server/handlers/base_handler.py @@ -125,6 +125,7 @@ def initialize(self, app): self.tabpy_state = app.tabpy_state # set content type to application/json self.set_header("Content-Type", "application/json") + self.protocol = self.settings[SettingsParameters.TransferProtocol] self.port = self.settings[SettingsParameters.Port] self.python_service = app.python_service self.credentials = app.credentials diff --git a/tabpy-server/tabpy_server/handlers/evaluation_plane_handler.py b/tabpy-server/tabpy_server/handlers/evaluation_plane_handler.py index 8b61f0b1..2a5686b2 100644 --- a/tabpy-server/tabpy_server/handlers/evaluation_plane_handler.py +++ b/tabpy-server/tabpy_server/handlers/evaluation_plane_handler.py @@ -8,19 +8,21 @@ class RestrictedTabPy: - def __init__(self, port, logger, timeout): + def __init__(self, protocol, port, logger, timeout): + self.protocol = protocol self.port = port self.logger = logger self.timeout = timeout def query(self, name, *args, **kwargs): - url = f'http://localhost:{self.port}/query/{name}' + url = f'{self.protocol}://localhost:{self.port}/query/{name}' self.logger.log(logging.DEBUG, f'Querying {url}...') internal_data = {'data': args or kwargs} data = json.dumps(internal_data) headers = {'content-type': 'application/json'} response = requests.post(url=url, data=data, headers=headers, - timeout=self.timeout) + timeout=self.timeout, + verify=False) return response.json() @@ -112,7 +114,10 @@ def post(self): @gen.coroutine def _call_subprocess(self, function_to_evaluate, arguments): restricted_tabpy = RestrictedTabPy( - self.port, self.logger, self.eval_timeout) + self.protocol, + self.port, + self.logger, + self.eval_timeout) # Exec does not run the function, so it does not block. exec(function_to_evaluate, globals()) diff --git a/tests/integration/integ_test_base.py b/tests/integration/integ_test_base.py index 0f1ed47f..11b0b2a9 100755 --- a/tests/integration/integ_test_base.py +++ b/tests/integration/integ_test_base.py @@ -141,13 +141,15 @@ def _get_key_file_name(self) -> str: def _get_evaluate_timeout(self) -> str: ''' Returns the configured timeout for the /evaluate method. - Default implementation returns None, which means that the timeout will default to 30. + Default implementation returns None, which means that + the timeout will default to 30. Returns ------- str Timeout for calling /evaluate. - If None, defaults TABPY_EVALUATE_TIMEOUT setting will default to '30'. + If None, defaults TABPY_EVALUATE_TIMEOUT setting + will default to '30'. ''' return None diff --git a/tests/integration/test_deploy_and_evaluate_model_ssl.py b/tests/integration/test_deploy_and_evaluate_model_ssl.py new file mode 100755 index 00000000..669dc724 --- /dev/null +++ b/tests/integration/test_deploy_and_evaluate_model_ssl.py @@ -0,0 +1,41 @@ +import integ_test_base +import requests +import subprocess +from pathlib import Path + + +class TestDeployAndEvaluateModelSSL(integ_test_base.IntegTestBase): + def _get_port(self): + return '9005' + + def _get_transfer_protocol(self) -> str: + return 'https' + + def _get_certificate_file_name(self) -> str: + return './tests/integration/resources/2019_04_24_to_3018_08_25.crt' + + def _get_key_file_name(self) -> str: + return './tests/integration/resources/2019_04_24_to_3018_08_25.key' + + def test_deploy_and_evaluate_model_ssl(self): + path = str(Path('models', 'setup.py')) + subprocess.call([self.py, path, self._get_config_file_name()]) + + payload = ( + '''{ + "data": { "_arg1": ["happy", "sad", "neutral"] }, + "script": + "return tabpy.query('Sentiment%20Analysis',_arg1)['response']" + }''') + + session = requests.Session() + # Do not verify servers' cert to be signed by trusted CA + session.verify = False + # Do not warn about insecure request + requests.packages.urllib3.disable_warnings() + response = session.post( + f'{self._get_transfer_protocol()}://' + f'localhost:{self._get_port()}/evaluate', + data=payload) + + self.assertEqual(200, response.status_code) diff --git a/tests/unit/server_tests/test_config.py b/tests/unit/server_tests/test_config.py index 38b7acf9..376cc98a 100644 --- a/tests/unit/server_tests/test_config.py +++ b/tests/unit/server_tests/test_config.py @@ -108,7 +108,9 @@ def test_config_file_present(self, mock_os, mock_path_exists, @patch('tabpy_server.app.app.os.path.exists', return_value=True) @patch('tabpy_server.app.app._get_state_from_file') @patch('tabpy_server.app.app.TabPyState') - def test_custom_evaluate_timeout_valid(self, mock_state, mock_get_state_from_file, mock_path_exists): + def test_custom_evaluate_timeout_valid(self, mock_state, + mock_get_state_from_file, + mock_path_exists): self.assertTrue(self.config_file is not None) config_file = self.config_file config_file.write('[TabPy]\n' @@ -121,7 +123,9 @@ def test_custom_evaluate_timeout_valid(self, mock_state, mock_get_state_from_fil @patch('tabpy_server.app.app.os.path.exists', return_value=True) @patch('tabpy_server.app.app._get_state_from_file') @patch('tabpy_server.app.app.TabPyState') - def test_custom_evaluate_timeout_invalid(self, mock_state, mock_get_state_from_file, mock_path_exists): + def test_custom_evaluate_timeout_invalid(self, mock_state, + mock_get_state_from_file, + mock_path_exists): self.assertTrue(self.config_file is not None) config_file = self.config_file config_file.write('[TabPy]\n' From ab3f7fc37f0c09bc096d697bb793d2c39e18d53c Mon Sep 17 00:00:00 2001 From: Oleksandr Golovatyi Date: Mon, 22 Jul 2019 13:17:53 -0700 Subject: [PATCH 07/28] Delete 'Supported Configurations' page (#327) --- README.md | 1 - docs/server-configurations.md | 23 ----------------------- 2 files changed, 24 deletions(-) delete mode 100644 docs/server-configurations.md diff --git a/README.md b/README.md index 5e169ae3..8c5752ef 100755 --- a/README.md +++ b/README.md @@ -20,7 +20,6 @@ order: * [TabPy Server Download Instructions](docs/server-download.md) * [TabPy Server Configuration Instructions](docs/server-config.md) * [TabPy Server Startup Instructions](docs/server-startup.md) -* [Supported TabPy Server Configurations](docs/server-configurations.md) * [Running TabPy in Python Virtual Environment](docs/tabpy-virtualenv.md) * [Authoring Python calculations in Tableau](docs/TableauConfiguration.md). * [TabPy Tools](docs/tabpy-tools.md) diff --git a/docs/server-configurations.md b/docs/server-configurations.md deleted file mode 100644 index 768967ba..00000000 --- a/docs/server-configurations.md +++ /dev/null @@ -1,23 +0,0 @@ -# Supported TabPy Server Configurations - -The following table lists environment configurations that have been -validated to support hosting TabPy instances. -To download specific release of TabPy find it on -[TabPy releases](https://github.com/tableau/TabPy/releases/) page. - - TabPy release | Python | Operating System | Owner | When confirmed to work | Comments --------------- |------- |----------------- |------ |----------------------- |---------- -0.4.1 | 3.6.5 | Windows 10 | @tableau | 2019-05-02 | Win 10 x64, Python 3.7.2 x64 -0.4.1 | 3.6.5 | centOS 7.6-1810 | @tableau | 2019-05-02 | -0.4.1 | 3.6.5 | macOS Sierra | @tableau | 2019-05-02 | -0.3.2 | 3.6.5 | Windows 10 | @tableau | 2019-01-29 | Win 10 x64, Python 3.5.6 x64 -0.3.2 | 3.6.5 | centOS 7.6-1810 | @tableau | 2019-01-30 | -0.3.2 | 3.6.5 | macOS Sierra | @tableau | 2019-01-29 | - -The following table lists environment configurations that have been -tested with TabPy instances, but are not guaranteed to be 100% supported. - - TabPy release | Python | Operating System | Owner | When confirmed to work | Comments --------------- |------- |----------------- |------ |----------------------- |---------- -0.4.1 | 3.6.7 | Ubuntu 18.04 | @tableau | 2019-05-03 | must be run as sudo -0.3.2 | 3.6.7 | Ubuntu 18.04 | @tableau | 2019-03-26 | Ubuntu ships Python 3.6.7 From 73979172677ac2a0144539d24936b966d22478e2 Mon Sep 17 00:00:00 2001 From: Oleksandr Golovatyi Date: Wed, 31 Jul 2019 11:10:29 -0700 Subject: [PATCH 08/28] Upgrade to Tornado 6 (#328) * remove asyncronous attribute for coroutines * Remove dependency on jsonschema * Fix markdown * Update setup.py * Add tornado to travis script * Add tornado to travis script * Remove tornado-json dependency * Update VERSION * Update CHANGELOG --- .travis.yml | 4 +--- CHANGELOG | 10 ++++++++++ README.md | 1 + VERSION | 2 +- docs/FAQ.md | 3 ++- tabpy-server/setup.py | 4 +--- tabpy-server/tabpy_server/app/app.py | 5 +---- tabpy-server/tabpy_server/handlers/endpoint_handler.py | 2 -- .../tabpy_server/handlers/endpoints_handler.py | 1 - .../tabpy_server/handlers/query_plane_handler.py | 5 +++-- tabpy-tools/setup.py | 3 +-- tabpy-tools/tabpy_tools/schema.py | 2 +- 12 files changed, 22 insertions(+), 20 deletions(-) diff --git a/.travis.yml b/.travis.yml index 61e236ca..b61f47f2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,9 +3,7 @@ language: python python: 3.6 install: - ./startup.sh --no-startup --print-install-logs - - pip install pytest - - pip install pytest-cov - - pip install coveralls + - pip install pytest pytest-cov coveralls - npm install -g markdownlint-cli script: - source utils/set_env.sh diff --git a/CHANGELOG b/CHANGELOG index 5b580214..58622042 100755 --- a/CHANGELOG +++ b/CHANGELOG @@ -3,6 +3,16 @@ This file lists notable changes for TabPy project releases. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## v0.7 + +### Improvements + +- Added t-test model +- Fixed models call with /evaluate for HTTPS +- Migrated to Tornado 6 +- Timeout is configurable with TABPY_EVALUATE_TIMEOUT config + file option + ## v0.6 ### Improvements diff --git a/README.md b/README.md index 8c5752ef..48f4e42d 100755 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ order: * [TabPy Tools](docs/tabpy-tools.md) Troubleshooting: + * [FAQ for configuration, startup and other issues](docs/FAQ.md) More technical topics: diff --git a/VERSION b/VERSION index 5a2a5806..eb49d7c7 100755 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.6 +0.7 diff --git a/docs/FAQ.md b/docs/FAQ.md index 4bd7b029..f50dd533 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -11,4 +11,5 @@ ### AttributeError: module 'tornado.web' has no attribute 'asynchronous' -TabPy uses Tornado 5.1.1. To it to your Python environment run `pip install tornado==5.1.1` and then try to start TabPy again. +TabPy uses Tornado 5.1.1. To it to your Python environment run +`pip install tornado==5.1.1` and then try to start TabPy again. diff --git a/tabpy-server/setup.py b/tabpy-server/setup.py index 17f8c643..8da8a3c0 100644 --- a/tabpy-server/setup.py +++ b/tabpy-server/setup.py @@ -38,7 +38,6 @@ 'decorator', 'future', 'genson', - 'jsonschema~=2.3.0', 'mock', 'numpy', 'pyopenssl', @@ -46,8 +45,7 @@ 'requests', 'singledispatch', 'six', - 'tornado==5.1.1', - 'Tornado-JSON', + 'tornado', 'urllib3<1.25,>=1.21.1' ] ) diff --git a/tabpy-server/tabpy_server/app/app.py b/tabpy-server/tabpy_server/app/app.py index 0cdc53aa..3e91d1e5 100644 --- a/tabpy-server/tabpy_server/app/app.py +++ b/tabpy-server/tabpy_server/app/app.py @@ -19,7 +19,6 @@ EvaluationPlaneHandler, QueryPlaneHandler, ServiceInfoHandler, StatusHandler, UploadDestinationHandler) -from tornado_json.constants import TORNADO_MAJOR import tornado @@ -251,9 +250,7 @@ def set_parameter(settings_key, config=tabpy_state, settings=self.settings) self.python_service = PythonServiceHandler(PythonService()) - self.settings['compress_response'] = True if TORNADO_MAJOR >= 4\ - else "gzip" - + self.settings['compress_response'] = True set_parameter(SettingsParameters.StaticPath, ConfigParameters.TABPY_STATIC_PATH, default_val='./') diff --git a/tabpy-server/tabpy_server/handlers/endpoint_handler.py b/tabpy-server/tabpy_server/handlers/endpoint_handler.py index f016c955..cfbf5957 100644 --- a/tabpy-server/tabpy_server/handlers/endpoint_handler.py +++ b/tabpy-server/tabpy_server/handlers/endpoint_handler.py @@ -42,7 +42,6 @@ def get(self, endpoint_name): self.error_out(404, 'Unknown endpoint', info=f'Endpoint {endpoint_name} is not found') - @tornado.web.asynchronous @gen.coroutine def put(self, name): if self.should_fail_with_not_authorized(): @@ -94,7 +93,6 @@ def put(self, name): self.error_out(500, err_msg) self.finish() - @tornado.web.asynchronous @gen.coroutine def delete(self, name): if self.should_fail_with_not_authorized(): diff --git a/tabpy-server/tabpy_server/handlers/endpoints_handler.py b/tabpy-server/tabpy_server/handlers/endpoints_handler.py index 6d1fc6d3..8745e457 100644 --- a/tabpy-server/tabpy_server/handlers/endpoints_handler.py +++ b/tabpy-server/tabpy_server/handlers/endpoints_handler.py @@ -26,7 +26,6 @@ def get(self): self._add_CORS_header() self.write(json.dumps(self.tabpy_state.get_endpoints())) - @tornado.web.asynchronous @gen.coroutine def post(self): if self.should_fail_with_not_authorized(): diff --git a/tabpy-server/tabpy_server/handlers/query_plane_handler.py b/tabpy-server/tabpy_server/handlers/query_plane_handler.py index baf25925..1950cbf7 100644 --- a/tabpy-server/tabpy_server/handlers/query_plane_handler.py +++ b/tabpy-server/tabpy_server/handlers/query_plane_handler.py @@ -9,6 +9,7 @@ from tabpy_server.common.util import format_exception import urllib import tornado.web +from tornado import gen def _get_uuid(): @@ -207,7 +208,7 @@ def _get_actual_model(self, endpoint_name): return (endpoint_name, all_endpoint_names) - @tornado.web.asynchronous + @gen.coroutine def get(self, endpoint_name): if self.should_fail_with_not_authorized(): self.fail_with_not_authorized() @@ -217,7 +218,7 @@ def get(self, endpoint_name): endpoint_name = urllib.parse.unquote(endpoint_name) self._process_query(endpoint_name, start) - @tornado.web.asynchronous + @gen.coroutine def post(self, endpoint_name): self.logger.log(logging.DEBUG, f'Processing POST for /query/{endpoint_name}...') diff --git a/tabpy-tools/setup.py b/tabpy-tools/setup.py index d18bf4bc..63a9ae64 100755 --- a/tabpy-tools/setup.py +++ b/tabpy-tools/setup.py @@ -25,7 +25,6 @@ install_requires=[ 'cloudpickle', 'requests', - 'genson', - 'jsonschema' + 'genson' ] ) diff --git a/tabpy-tools/tabpy_tools/schema.py b/tabpy-tools/tabpy_tools/schema.py index cba12df8..f001d7f0 100755 --- a/tabpy-tools/tabpy_tools/schema.py +++ b/tabpy-tools/tabpy_tools/schema.py @@ -1,7 +1,7 @@ import logging import genson as _genson -from jsonschema import validate as _validate +from json import validate as _validate logger = logging.getLogger(__name__) From f0601dc06464f61db514ec714ff9488b959cbe22 Mon Sep 17 00:00:00 2001 From: ogolovatyi Date: Wed, 31 Jul 2019 11:20:29 -0700 Subject: [PATCH 09/28] Fix pycodestyle warnings --- models/utils/setup_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/models/utils/setup_utils.py b/models/utils/setup_utils.py index e3da48de..a5550257 100644 --- a/models/utils/setup_utils.py +++ b/models/utils/setup_utils.py @@ -33,8 +33,9 @@ def get_creds(): passwd = sys.stdin.readline().rstrip() return [user, passwd] + def main(funcName, func, funcDescription): - # running from setup.py + # running from setup.py if len(sys.argv) > 1: config_file_path = sys.argv[1] else: From 8fb651106bb434b632fd8dbf658590d233dcaa79 Mon Sep 17 00:00:00 2001 From: Oleksandr Golovatyi Date: Wed, 14 Aug 2019 16:37:22 -0700 Subject: [PATCH 10/28] Make TabPy pip package (#330) * move server and tools one folder up * Tabpy folder for a package * tabpy * Unit tests passing * integration tests passing * fix codestyle * fix markdown * Include data files in dist * Update documentation * Update travis build steps * Update travis build steps * Update travis build steps * Update travis build steps * Fixing tests.... * Fixing tests.... * Fixing tests.... * Fixing tests.... * Fixing tests.... * Fixing tests.... * Fixing tests.... * Fix pip install instructions * Add unit test for generate_schema * Documentation fixes * Models moved under tabpy, version fixed * Change startup scripts to tell about pip tabpy * Update documentation for models * Fix typos * fix markdown * Update changelog * Add user-management as a command * Add user-management as a command * Fix format_exception reference * Create state.ini from template * Make tabpy.utils a module * Fixing typos in documentation * Fix unit tests * Deploy test to use custom config * Delete state.ini * Delete state.ini * Fixing code coverage settings * Fixing code coverage settings * Add code coverage for integration tests * Fix ignored tests to pass --- .coveragerc | 9 +- .gitattributes | 1 - .gitignore | 7 +- .travis.yml | 7 +- .vscode/launch.json | 16 -- .vscode/settings.json | 22 -- CHANGELOG | 10 +- CONTRIBUTING.md | 81 ++++---- MANIFEST.in | 7 + README.md | 6 +- VERSION | 2 +- docs/FAQ.md | 4 + docs/server-config.md | 188 ++++++++++++------ docs/server-download.md | 13 -- docs/server-install.md | 49 +++++ docs/server-startup.md | 127 ------------ docs/tabpy-tools.md | 55 ++--- docs/tabpy-virtualenv.md | 55 ++--- setup.py | 94 +++++++++ startup.cmd | 88 +------- startup.sh | 116 +---------- tabpy-server/setup.py | 51 ----- tabpy-server/tabpy_server/__init__.py | 18 -- .../tabpy_server/handlers/__init__.py | 13 -- tabpy-server/tabpy_server/tabpy.py | 10 - tabpy-tools/setup.py | 30 --- tabpy-tools/tabpy_tools/__init__.py | 23 --- {models/utils => tabpy}/__init__.py | 0 .../app => tabpy/models}/__init__.py | 0 .../setup.py => tabpy/models/deploy_models.py | 7 +- {models => tabpy/models}/scripts/PCA.py | 10 +- .../models}/scripts/SentimentAnalysis.py | 17 +- .../models/scripts}/__init__.py | 0 {models => tabpy/models}/scripts/tTest.py | 10 +- .../models/utils}/__init__.py | 0 {models => tabpy/models}/utils/setup_utils.py | 21 +- tabpy/tabpy.py | 31 +++ .../psws => tabpy/tabpy_server}/__init__.py | 0 .../tabpy_server/app/ConfigParameters.py | 0 .../tabpy_server/app/SettingsParameters.py | 0 tabpy/tabpy_server/app/__init__.py | 0 .../tabpy_server/app/app.py | 82 +++++--- .../tabpy_server/app/util.py | 0 tabpy/tabpy_server/common/__init__.py | 0 .../tabpy_server/common/default.conf | 10 +- .../tabpy_server/common/endpoint_file_mgr.py | 0 .../tabpy_server/common/messages.py | 0 .../tabpy_server/common/util.py | 0 tabpy/tabpy_server/handlers/__init__.py | 13 ++ .../tabpy_server/handlers/base_handler.py | 4 +- .../tabpy_server/handlers/endpoint_handler.py | 16 +- .../handlers/endpoints_handler.py | 8 +- .../handlers/evaluation_plane_handler.py | 4 +- .../tabpy_server/handlers/main_handler.py | 2 +- .../handlers/management_handler.py | 10 +- .../handlers/query_plane_handler.py | 6 +- .../handlers/service_info_handler.py | 4 +- .../tabpy_server/handlers/status_handler.py | 2 +- .../handlers/upload_destination_handler.py | 4 +- .../tabpy_server/handlers/util.py | 2 +- tabpy/tabpy_server/management/__init__.py | 0 .../tabpy_server/management/state.py | 2 +- .../tabpy_server/management/util.py | 4 +- tabpy/tabpy_server/psws/__init__.py | 0 .../tabpy_server/psws/callbacks.py | 15 +- .../tabpy_server/psws/python_service.py | 6 +- .../tabpy_server/state.ini.template | 3 +- .../tabpy_server/static/index.html | 0 .../tabpy_server/static/tableau.png | Bin tabpy/tabpy_tools/__init__.py | 0 {tabpy-tools => tabpy}/tabpy_tools/client.py | 0 .../tabpy_tools/custom_query_object.py | 0 .../tabpy_tools/query_object.py | 0 {tabpy-tools => tabpy}/tabpy_tools/rest.py | 3 + .../tabpy_tools/rest_client.py | 0 {tabpy-tools => tabpy}/tabpy_tools/schema.py | 5 +- tabpy/utils/__init__.py | 0 {utils => tabpy/utils}/user_management.py | 8 +- tests/integration/integ_test_base.py | 33 ++- .../resources/deploy_and_evaluate_model.conf | 57 ++++++ .../test_deploy_and_evaluate_model.py | 15 +- .../test_deploy_and_evaluate_model_ssl.py | 3 +- .../test_deploy_model_ssl_off_auth_off.py | 6 +- .../test_deploy_model_ssl_off_auth_on.py | 7 +- .../test_deploy_model_ssl_on_auth_off.py | 5 +- .../test_deploy_model_ssl_on_auth_on.py | 12 +- tests/unit/server_tests/test_config.py | 87 ++++---- .../test_endpoint_file_manager.py | 2 +- .../server_tests/test_endpoint_handler.py | 6 +- .../server_tests/test_endpoints_handler.py | 6 +- .../test_evaluation_plane_handler.py | 6 +- tests/unit/server_tests/test_pwd_file.py | 2 +- .../server_tests/test_service_info_handler.py | 6 +- tests/unit/tools_tests/test_client.py | 2 +- tests/unit/tools_tests/test_rest.py | 31 ++- tests/unit/tools_tests/test_rest_object.py | 2 +- tests/unit/tools_tests/test_schema.py | 41 ++++ utils/set_env.cmd | 2 - utils/set_env.sh | 1 - 99 files changed, 835 insertions(+), 908 deletions(-) delete mode 100755 .gitattributes delete mode 100755 .vscode/launch.json delete mode 100755 .vscode/settings.json create mode 100755 MANIFEST.in delete mode 100755 docs/server-download.md create mode 100755 docs/server-install.md delete mode 100755 docs/server-startup.md create mode 100755 setup.py delete mode 100644 tabpy-server/setup.py delete mode 100644 tabpy-server/tabpy_server/__init__.py delete mode 100644 tabpy-server/tabpy_server/handlers/__init__.py delete mode 100644 tabpy-server/tabpy_server/tabpy.py delete mode 100755 tabpy-tools/setup.py delete mode 100755 tabpy-tools/tabpy_tools/__init__.py rename {models/utils => tabpy}/__init__.py (100%) mode change 100644 => 100755 rename {tabpy-server/tabpy_server/app => tabpy/models}/__init__.py (100%) mode change 100644 => 100755 rename models/setup.py => tabpy/models/deploy_models.py (95%) rename {models => tabpy/models}/scripts/PCA.py (90%) rename {models => tabpy/models}/scripts/SentimentAnalysis.py (79%) rename {tabpy-server/tabpy_server/common => tabpy/models/scripts}/__init__.py (100%) mode change 100644 => 100755 rename {models => tabpy/models}/scripts/tTest.py (85%) rename {tabpy-server/tabpy_server/management => tabpy/models/utils}/__init__.py (100%) rename {models => tabpy/models}/utils/setup_utils.py (79%) create mode 100755 tabpy/tabpy.py rename {tabpy-server/tabpy_server/psws => tabpy/tabpy_server}/__init__.py (100%) rename {tabpy-server => tabpy}/tabpy_server/app/ConfigParameters.py (100%) rename {tabpy-server => tabpy}/tabpy_server/app/SettingsParameters.py (100%) create mode 100644 tabpy/tabpy_server/app/__init__.py rename {tabpy-server => tabpy}/tabpy_server/app/app.py (84%) rename {tabpy-server => tabpy}/tabpy_server/app/util.py (100%) create mode 100644 tabpy/tabpy_server/common/__init__.py rename {tabpy-server => tabpy}/tabpy_server/common/default.conf (86%) rename {tabpy-server => tabpy}/tabpy_server/common/endpoint_file_mgr.py (100%) rename {tabpy-server => tabpy}/tabpy_server/common/messages.py (100%) rename {tabpy-server => tabpy}/tabpy_server/common/util.py (100%) create mode 100644 tabpy/tabpy_server/handlers/__init__.py rename {tabpy-server => tabpy}/tabpy_server/handlers/base_handler.py (99%) rename {tabpy-server => tabpy}/tabpy_server/handlers/endpoint_handler.py (93%) rename {tabpy-server => tabpy}/tabpy_server/handlers/endpoints_handler.py (95%) rename {tabpy-server => tabpy}/tabpy_server/handlers/evaluation_plane_handler.py (97%) rename {tabpy-server => tabpy}/tabpy_server/handlers/main_handler.py (70%) rename {tabpy-server => tabpy}/tabpy_server/handlers/management_handler.py (94%) rename {tabpy-server => tabpy}/tabpy_server/handlers/query_plane_handler.py (97%) rename {tabpy-server => tabpy}/tabpy_server/handlers/service_info_handler.py (86%) rename {tabpy-server => tabpy}/tabpy_server/handlers/status_handler.py (93%) rename {tabpy-server => tabpy}/tabpy_server/handlers/upload_destination_handler.py (79%) rename {tabpy-server => tabpy}/tabpy_server/handlers/util.py (89%) create mode 100644 tabpy/tabpy_server/management/__init__.py rename {tabpy-server => tabpy}/tabpy_server/management/state.py (99%) rename {tabpy-server => tabpy}/tabpy_server/management/util.py (93%) create mode 100644 tabpy/tabpy_server/psws/__init__.py rename {tabpy-server => tabpy}/tabpy_server/psws/callbacks.py (93%) rename {tabpy-server => tabpy}/tabpy_server/psws/python_service.py (98%) rename tabpy-server/tabpy_server/state.ini => tabpy/tabpy_server/state.ini.template (86%) rename {tabpy-server => tabpy}/tabpy_server/static/index.html (100%) rename {tabpy-server => tabpy}/tabpy_server/static/tableau.png (100%) create mode 100755 tabpy/tabpy_tools/__init__.py rename {tabpy-tools => tabpy}/tabpy_tools/client.py (100%) rename {tabpy-tools => tabpy}/tabpy_tools/custom_query_object.py (100%) rename {tabpy-tools => tabpy}/tabpy_tools/query_object.py (100%) rename {tabpy-tools => tabpy}/tabpy_tools/rest.py (98%) rename {tabpy-tools => tabpy}/tabpy_tools/rest_client.py (100%) rename {tabpy-tools => tabpy}/tabpy_tools/schema.py (98%) create mode 100755 tabpy/utils/__init__.py rename {utils => tabpy/utils}/user_management.py (94%) create mode 100755 tests/integration/resources/deploy_and_evaluate_model.conf create mode 100755 tests/unit/tools_tests/test_schema.py delete mode 100755 utils/set_env.cmd delete mode 100755 utils/set_env.sh diff --git a/.coveragerc b/.coveragerc index c82dc401..228ef0f8 100755 --- a/.coveragerc +++ b/.coveragerc @@ -4,4 +4,11 @@ exclude_lines = if __name__ == .__main__.: # Only show one number after decimal point in report. -precision = 1 \ No newline at end of file +precision = 1 + +[run] +omit = + tabpy/models/* + tabpy/tabpy.py + tabpy/utils/* + tests/* diff --git a/.gitattributes b/.gitattributes deleted file mode 100755 index 8f5277c3..00000000 --- a/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -tabpy-server/tabpy_server/_version.py export-subst diff --git a/.gitignore b/.gitignore index a204dcab..621c6072 100644 --- a/.gitignore +++ b/.gitignore @@ -117,12 +117,13 @@ package-lock.json .idea/ # TabPy server artifacts -tabpy-server/install.log -tabpy-server/tabpy_server/query_objects -tabpy-server/tabpy_server/staging +tabpy/tabpy_server/state.ini +tabpy/tabpy_server/query_objects +tabpy/tabpy_server/staging # VS Code *.code-workspace +.vscode # etc setup.bat diff --git a/.travis.yml b/.travis.yml index b61f47f2..13cbf053 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,13 +2,12 @@ os: linux language: python python: 3.6 install: - - ./startup.sh --no-startup --print-install-logs - pip install pytest pytest-cov coveralls - npm install -g markdownlint-cli script: - - source utils/set_env.sh - - pytest tests/unit --cov=tabpy-server/tabpy_server --cov=tabpy-tools/tabpy_tools --cov-append - - pytest tests/integration + - pip install -e . + - py.test tests/unit --cov=tabpy --cov-append + - py.test tests/integration --cov=tabpy --cov-append - markdownlint . after_success: - coveralls diff --git a/.vscode/launch.json b/.vscode/launch.json deleted file mode 100755 index 6c044293..00000000 --- a/.vscode/launch.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "name": "Python: General", - "type": "python", - "request": "launch", - "program": "${file}", - "console": "externalTerminal", - "env": {"${PYTHONPATH}": "${PYTHONPATH};${workspaceRoot}/tabpy-server;${workspaceRoot}/tabpy-tools"} - } - ] -} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100755 index 4a047b81..00000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "git.enabled": true, - "files.exclude": { - "**/build": true, - "**/dist": true, - "**/__pycache__": true, - "**/.pytest_cache": true, - "**/*.egg-info": true, - "**/*.pyc": true - }, - "python.linting.pylintEnabled": false, - "python.linting.flake8Enabled": false, - "python.linting.enabled": true, - "python.testing.autoTestDiscoverOnSaveEnabled": true, - "python.testing.pytestArgs": [ - "." - ], - "python.testing.unittestEnabled": false, - "python.testing.nosetestsEnabled": false, - "python.testing.pytestEnabled": true, - "python.linting.pep8Enabled": true -} \ No newline at end of file diff --git a/CHANGELOG b/CHANGELOG index 3a8dc478..bd54d1c1 100755 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,7 +1,11 @@ -# TabPy Changelog +# Changelog -This file lists notable changes for TabPy project releases. -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## v0.8 + +### Improvements + +- TabPy is pip package now +- Models are deployed with updated script ## v0.7 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index adb3f35e..6f9702cb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,21 +1,26 @@ # TabPy Contributing Guide + + - [Environment Setup](#environment-setup) - [Prerequisites](#prerequisites) - [Cloning TabPy Repository](#cloning-tabpy-repository) -- [Setting Up Environment](#setting-up-environment) -- [Unit Tests](#unit-tests) -- [Integration Tests](#integration-tests) +- [Tests](#tests) + * [Unit Tests](#unit-tests) + * [Integration Tests](#integration-tests) - [Code Coverage](#code-coverage) - [TabPy in Python Virtual Environment](#tabpy-in-python-virtual-environment) - [Documentation Updates](#documentation-updates) - [TabPy with Swagger](#tabpy-with-swagger) - [Code styling](#code-styling) +- [Publishing TabPy Package](#publishing-tabpy-package) + + ## Environment Setup The purpose of this guide is to enable developers of Tabpy to install the project @@ -27,12 +32,18 @@ These are prerequisites for an environment required for a contributor to be able to work on TabPy changes: - Python 3.6.5: - - To see which version of Python you have installed, run ```python --version```. + - To see which version of Python you have installed, run `python --version`. - git - TabPy repo: - - Get the latest TabPy repository with `git clone https://github.com/tableau/TabPy.git` + - Get the latest TabPy repository with + `git clone https://github.com/tableau/TabPy.git`. - Create a new branch for your changes. - When changes are ready push them on github and create merge request. +- PIP packages - install all with + `pip install pytest pycodestyle autopep8 twine --upgrade` command +- Node.js for npm packages - install from . +- NPM packages - install all with + `npm install markdown-toc markdownlint` command. ## Cloning TabPy Repository @@ -46,32 +57,29 @@ be able to work on TabPy changes: cd TabPy ``` -Before making any code changes run environment setup script. -For Windows run this command from the repository root folder: +4. Register TabPy repo as a pip package: -```sh -utils\set_env.cmd -``` + ```sh + pip install -e . + ``` -and for Linux or Mac the next command from the repository root folder: +## Tests + +To run the whole test suite execute the following command: ```sh -source utils/set_env.sh +pytest ``` -## Unit Tests +### Unit Tests -TabPy has test suites for `tabpy-server` and `tabpy-tools` components. -To run the unit tests use `pytest` which you may need to install first -(see [https://docs.pytest.org](https://docs.pytest.org) for details): +Unit tests suite can be exectud with the following command: ```sh pytest tests/unit ``` -Check `pytest` documentation for how to run individual tests or set of tests. - -## Integration Tests +### Integration Tests Integration tests can be executed with the next command: @@ -106,13 +114,19 @@ TOC for markdown file is built with [markdown-toc](https://www.npmjs.com/package markdown-toc -i docs/server-startup.md ``` +To check markdown style for all the documentation use `markdownlint`: + +```sh +markdownlint . +``` + These checks will run as part of the build if you submit a pull request. ## TabPy with Swagger You can invoke the TabPy Server API against a running TabPy instance with Swagger. -- Make CORS related changes in TabPy configuration file: update `tabpy-server\state.ini` +- Make CORS related changes in TabPy configuration file: update `tabpy/tabpy-server/state.ini` file in your local repository to have the next settings: ```config @@ -131,32 +145,29 @@ Access-Control-Allow-Methods = GET, OPTIONS, POST ## Code styling -`pycodestyle` is used to check Python code against our style conventions. -You can run install it and run locally for files where modifications were made: +`pycodestyle` is used to check Python code against our style conventions: ```sh -pip install pycodestyle -``` - -And then run it for files where modifications were made, e.g.: - -```sh -pycodestyle tabpy-server/server_tests/test_pwd_file.py +pycodestyle . ``` For reported errors and warnings either fix them manually or auto-format files with `autopep8`. -To install `autopep8` run the next command: +Run the tool for a file. In the example below `-i` +option tells `autopep8` to update the file. Without the option it +outputs formatted code to the console. ```sh -pip install autopep8 +autopep8 -i tabpy-server/server_tests/test_pwd_file.py ``` -Then you can run the tool for a file. In the example below `-i` -option tells `autopep8` to update the file. Without the option it -outputs formatted code to the console. +## Publishing TabPy Package + +Execute the following commands to build and publish new version of +TabPy package: ```sh -autopep8 -i tabpy-server/server_tests/test_pwd_file.py +python setup.py sdist bdist_wheel +python -m twine upload dist/* ``` diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100755 index 00000000..0caceefb --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,7 @@ +include \ + CHANGELOG \ + LICENSE \ + VERSION \ + tabpy/tabpy_server/state.ini \ + tabpy/tabpy_server/static/* \ + tabpy/tabpy_server/common/default.conf \ diff --git a/README.md b/README.md index 65bbce07..a744f703 100755 --- a/README.md +++ b/README.md @@ -14,13 +14,11 @@ TabPy (the Tableau Python Server) is an external service implementation which ex Tableau's capabilities by allowing users to execute Python scripts and saved functions via Tableau's table calculations. -All documentation is in the [docs](docs) folder. Consider reading it in this -order: +Consider reading TabPy documentation in the following order: * [About TabPy](docs/about.md) -* [TabPy Server Download Instructions](docs/server-download.md) +* [TabPy Installation Instructions](docs/server-install.md) * [TabPy Server Configuration Instructions](docs/server-config.md) -* [TabPy Server Startup Instructions](docs/server-startup.md) * [Running TabPy in Python Virtual Environment](docs/tabpy-virtualenv.md) * [Authoring Python calculations in Tableau](docs/TableauConfiguration.md). * [TabPy Tools](docs/tabpy-tools.md) diff --git a/VERSION b/VERSION index 0e2c9395..f83dbb32 100755 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.7 \ No newline at end of file +0.7.8 \ No newline at end of file diff --git a/docs/FAQ.md b/docs/FAQ.md index f50dd533..77044de7 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -3,6 +3,10 @@ + +- [Startup Issues](#startup-issues) + * [AttributeError: module 'tornado.web' has no attribute 'asynchronous'](#attributeerror-module-tornadoweb-has-no-attribute-asynchronous) + diff --git a/docs/server-config.md b/docs/server-config.md index a57bd363..1bd445dd 100755 --- a/docs/server-config.md +++ b/docs/server-config.md @@ -4,47 +4,149 @@ +- [Custom Settings](#custom-settings) + * [Configuration File Content](#configuration-file-content) + * [Configuration File Example](#configuration-file-example) - [Configuring HTTP vs HTTPS](#configuring-http-vs-https) - [Authentication](#authentication) * [Enabling Authentication](#enabling-authentication) * [Password File](#password-file) - * [Setting Up Environment](#setting-up-environment) * [Adding an Account](#adding-an-account) * [Updating an Account](#updating-an-account) * [Deleting an Account](#deleting-an-account) - [Logging](#logging) * [Request Context Logging](#request-context-logging) -- [Custom Script Timeout](#custom-script-timeout) -Default settings for TabPy may be viewed in the -tabpy_server/common/default.conf file. This file also contains -commented examples of how to set up your TabPy server to only -serve HTTPS traffic and enable authentication. +## Custom Settings -Change settings by: +TabPy starts with set of default settings unless settings are provided via +environment variables or with a config file. -1. Adding environment variables: - - set the environment variable as required by your Operating System. When - creating environment variables, use the same name as is in the config file - as an environment variable. The files startup.sh and startup.cmd in the root - of the install folder have examples of how to set environment variables in - both Linux and Windows respectively. Set any desired environment variables - and then start the application. -2. Modifying default.conf. -3. Specifying your own config file as a command line parameter. - - i.e. Running the command: +Configuration parameters can be updated with: - ```sh - python tabpy.py --config=path\to\my\config - ``` +1. Adding environment variables - set the environment variable as required by + your Operating System. When creating environment variables, use the same + name for your environment variable as specified in the config file. +2. Specifying a parameter in a config file (enviroment variable value overwrites + configuration setting). + +Configuration file with custom settings is specified as a command line parameter: + +```sh +tabpy --config=path/to/my/config/file.conf +``` The default config file is provided to show you the default values but does not need to be present to run TabPy. +### Configuration File Content + +Configuration file consists of settings for TabPy itself and Python logger +settings. You should only set parameters if you need different values than +the defaults. + +TabPy parameters explained below, the logger documentation can be found +at [`logging.config` documentation page](https://docs.python.org/3.6/library/logging.config.html). + +`[TabPy]` parameters: + +- `TABPY_PORT` - port for TabPy to listen on. Default value - `9004`. +- `TABPY_QUERY_OBJECT_PATH` - query objects location. Used with models, see + [TabPy Tools documentation](tabpy-tools.md) for details. Default value - + `/tmp/query_objects`. +- `TABPY_STATE_PATH` - state location for Tornado web server. Default + value - `tabpy/tabpy_server` subfolder in TabPy package folder. +- `TABPY_STATIC_PATH` - location of static files (index.html page) for + TabPy instance. Default value - `tabpy/tabpy_server/static` subfolder in + TabPy package folder. +- `TABPY_PWD_FILE` - path to password file. Setting up this parameter + makes TabPy require credentials with HTTP(S) requests. More details about + authentication can be found in [Authentication](#authentication) + section. Default value - not set. +- `TABPY_TRANSFER_PROTOCOL` - transfer protocol. Default value - `http`. If + set to `https` two additional parameters have to be specified: + `TABPY_CERTIFICATE_FILE` and `TABPY_KEY_FILE`. More details for how to + configure TabPy for HTTPS are at [Configuring HTTP vs HTTPS] + (#configuring-http-vs-https) section. +- `TABPY_CERTIFICATE_FILE` the certificate file to run TabPy with. Only used + with `TABPY_TRANSFER_PROTOCOL` set to `https`. Default value - not set. +- `TABPY_KEY_FILE` to private key file to run TabPy with. Only used + with `TABPY_TRANSFER_PROTOCOL` set to `https`. Default value - not set. +- `TABPY_LOG_DETAILS` - when set to `true` additional call information + (caller IP, URL, client info, etc.) is logged. Default value - `false`. +- `TABPY_EVALUATE_TIMEOUT` - script evaluation timeout in seconds. Default + value - `30`. + +### Configuration File Example + +**Note:** _Always use absolute paths for the configuration paths +settings._ + +```ini +[TabPy] +# TABPY_QUERY_OBJECT_PATH = /tmp/query_objects +# TABPY_PORT = 9004 +# TABPY_STATE_PATH = /tabpy/tabpy_server + +# Where static pages live +# TABPY_STATIC_PATH = /tabpy/tabpy_server/static + +# For how to configure TabPy authentication read +# docs/server-config.md. +# TABPY_PWD_FILE = /path/to/password/file.txt + +# To set up secure TabPy uncomment and modify the following lines. +# Note only PEM-encoded x509 certificates are supported. +# TABPY_TRANSFER_PROTOCOL = https +# TABPY_CERTIFICATE_FILE = path/to/certificate/file.crt +# TABPY_KEY_FILE = path/to/key/file.key + +# Log additional request details including caller IP, full URL, client +# end user info if provided. +# TABPY_LOG_DETAILS = true + +# Configure how long a custom script provided to the /evaluate method +# will run before throwing a TimeoutError. +# The value should be a float representing the timeout time in seconds. +#TABPY_EVALUATE_TIMEOUT = 30 + +[loggers] +keys=root + +[handlers] +keys=rootHandler,rotatingFileHandler + +[formatters] +keys=rootFormatter + +[logger_root] +level=DEBUG +handlers=rootHandler,rotatingFileHandler +qualname=root +propagete=0 + +[handler_rootHandler] +class=StreamHandler +level=DEBUG +formatter=rootFormatter +args=(sys.stdout,) + +[handler_rotatingFileHandler] +class=handlers.RotatingFileHandler +level=DEBUG +formatter=rootFormatter +args=('tabpy_log.log', 'a', 1000000, 5) + +[formatter_rootFormatter] +format=%(asctime)s [%(levelname)s] (%(filename)s:%(module)s:%(lineno)d): %(message)s +datefmt=%Y-%m-%d,%H:%M:%S + +``` + ## Configuring HTTP vs HTTPS By default, TabPy serves only HTTP requests. TabPy can be configured to serve @@ -86,41 +188,25 @@ Password file is a text file containing usernames and hashed passwords per line separated by single space. For username only ASCII characters are supported. Usernames are case-insensitive. -Passwords in the password file are hashed with PBKDF2. [See source code -for implementation details](../tabpy-server/tabpy_server/handlers/util.py). +Passwords in the password file are hashed with PBKDF2. **It is highly recommended to restrict access to the password file with hosting OS mechanisms. Ideally the file should only be accessible for reading with the account under which TabPy runs and TabPy admin account.** -There is a `utils/user_management.py` utility to operate with -accounts in the password file. Run `utils/user_management.py -h` to -see how to use it. +There is a `tabpy-user-management` command provided with `tabpy` package to +operate with accounts in the password file. Run `tabpy-user-management -h` +to see how to use it. After making any changes to the password file, TabPy needs to be restarted. -### Setting Up Environment - -Before making any code changes run the environment setup script. For -Windows run this command from the repository root folder: - -```sh -utils\set_env.cmd -``` - -and for Linux or Mac run this command from the repository root folder: - -```sh -source utils/set_env.sh -``` - ### Adding an Account -To add an account run `utils/user_management.py` utility with `add` +To add an account run `tabpy-user-management add` command providing user name, password (optional) and password file: ```sh -python utils/user_management.py add -u -p -f +tabpy-user-management add -u -p -f ``` If the (recommended) `-p` argument is not provided a password for the user name @@ -128,11 +214,11 @@ will be generated and displayed in the command line. ### Updating an Account -To update the password for an account run `utils/user_management.py` utility -with `update` command: +To update the password for an account run `tabpy-user-management update` +command: ```sh -python utils/user_management.py update -u -p -f +tabpy-user-management update -u -p -f ``` If the (recommended) `-p` agrument is not provided a password for the user name @@ -193,15 +279,3 @@ arg1, _arg2): No passwords are logged. NOTE the request context details are logged with INFO level. - -## Custom Script Timeout - -By default, all custom scripts executed through `POST /evaluate` may run for up -to 30.0 s before being terminated. To configure this timeout, uncomment -`TABPY_EVALUATE_TIMEOUT = 30` in the default config under -`tabpy-server/tabpy_server/common/default.conf` and replace `30` with the float -value of your choice representing the timeout time in seconds, or add such an -entry to your custom config. - -This timeout does not apply when evaluating models either through the `/query` -method, or using the `tabpy.query(...)` syntax with the `/evaluate` method. diff --git a/docs/server-download.md b/docs/server-download.md deleted file mode 100755 index fe6f45fb..00000000 --- a/docs/server-download.md +++ /dev/null @@ -1,13 +0,0 @@ -# TabPy Server Download - -The TabPy server can be downloaded from -[TabPy github releases page](https://github.com/tableau/TabPy/releases). -Read the description for the releases and then download the appropriate -`zip` or `tar.gz` TabPy archive to your machine. The latest release -of TabPy is strongly recommended unless you have a specific restriction -or package incompatability you need to consider. - -After downloading the archive unpack it to a folder on your machine and -you are ready to start running TabPy from that folder. Read the -[configuration](server-config.md) and [starting up instructions](server-startup.md) -for more details. diff --git a/docs/server-install.md b/docs/server-install.md new file mode 100755 index 00000000..e91aa4dc --- /dev/null +++ b/docs/server-install.md @@ -0,0 +1,49 @@ +# TabPy Installation Instructions + +These instructions explain how to install and start up TabPy Server. + + + + + +- [TabPy Installation](#tabpy-installation) +- [Starting TabPy](#starting-tabpy) + + + + + +## TabPy Installation + +To install TabPy on to an environment `pip` needs to be installed and +updated first: + +```sh +python -m pip install --upgrade pip +``` + +Now TabPy can be install as a package: + +```sh +pip install tabpy +``` + +## Starting TabPy + +To start TabPy with default setting run the following command: + +```sh +tabpy +``` + +To run TabPy with custom settings create config file with parameters +explained in [TabPy Server Configuration Instructions](server-config.md) +and specify it in command line: + +```sh +tabpy --config=path/to/my/config/file.conf +``` + +It is highly recommended to use Python virtual enviroment for running TabPy. +Check the [Running TabPy in Python Virtual Environment](tabpy-virtualenv.md) page +for more details. diff --git a/docs/server-startup.md b/docs/server-startup.md deleted file mode 100755 index 3257afec..00000000 --- a/docs/server-startup.md +++ /dev/null @@ -1,127 +0,0 @@ -# Setup and Startup TabPy Server - -These instructions explain how to start up TabPy Server. - - - - -- [Prerequisites](#prerequisites) -- [Windows](#windows) - * [Command Line Arguments](#command-line-arguments) -- [Mac](#mac) - * [Command Line Arguments](#command-line-arguments-1) -- [Linux](#linux) - * [Command Line Arguments](#command-line-arguments-2) - - - - -## Prerequisites - -To start up TabPy Server from an environment the following prerequisites are required: - -- Python 3.6.5 -- setuptools (Python module, can be installed from PyPi) - -First, select a TabPy version and download its source code from the -[releases page](https://github.com/tableau/TabPy/releases). To start up -a TabPy server instance, follow the instructions for your OS (found below). - -Instructions on how to configure your TabPy server instance can be found in the -[TabPy Server Configuration Instructions](server-config.md) - -It is highly recommended to use Python virtual enviroment for running TabPy. -Check the [Running TabPy in Python Virtual Environment](tabpy-virtualenv.md) page -for more details. -If you are installing a newer version of TabPy in the same environment as a -previous install, delete the previous TabPy version folder in your Python directory. - -## Windows - -1. Open a command prompt. -2. Navigate to the folder in which you downloaded your source code. - - This folder should contain the file: ```startup.cmd``` -3. Run the following command from the command prompt: - - ```batch - startup.cmd - ``` - -### Command Line Arguments for Windows - -To specify the *config file* with which to configure your server instance, pass -it in as a command line argument as follows: - -```batch -startup.cmd myconfig.conf -``` - -Replace `myconfig.conf` with the path to your config file relative to -`%TABPY_ROOT%\tabpy-server\tabpy_server\`. - -For example, in this case your config file would be located at -`%TABPY_ROOT%\tabpy-server\tabpy_server\myconfig.conf`. - -## Mac - -1. Open a terminal. -2. Navigate to the folder in which you downloaded your source code. - - This folder should contain the file: ```startup.sh``` -3. Run the following command from the terminal: - - ```bash - ./startup.sh - ``` - -### Command Line Arguments for Mac - -- To specify the *config file* with which to configure your server instance, - set the ```--config=*``` or ```-c=*``` command line argument as follows: - - ```bash - ./startup.sh --config=myconfig.conf - ``` - - or - - ```bash - ./startup.sh -c=myconfig.conf - ``` - - Replace ```myconfig.conf``` with the path to your config file relative to - ```$TABPY_ROOT/tabpy-server/tabpy_server/```. - - For example, in this case your config file would be located at - ```$TABPY_ROOT/tabpy-server/tabpy_server/myconfig.conf```. - -## Linux - -1. Open a terminal. -2. Navigate to the folder in which you downloaded your source code. - - This folder should contain the file: ```startup.sh``` -3. Run the following command from the terminal: - - ```bash - ./startup.sh - ``` - -### Command Line Arguments for Linux - -- To specify the *config file* with which to configure your server instance, - set the ```--config=*``` or ```-c=*``` command line argument as follows: - - ```bash - ./startup.sh --config=myconfig.conf - ``` - - or - - ```bash - ./startup.sh -c=myconfig.conf - ``` - - Replace ```myconfig.conf``` with the path to your config file relative to - ```$TABPY_ROOT/tabpy-server/tabpy_server/```. - - For example, in this case your config file would be located at - ```$TABPY_ROOT/tabpy-server/tabpy_server/myconfig.conf```. diff --git a/docs/tabpy-tools.md b/docs/tabpy-tools.md index d46839f0..07ca73c4 100755 --- a/docs/tabpy-tools.md +++ b/docs/tabpy-tools.md @@ -3,18 +3,25 @@ TabPy tools is the Python package of tools for managing the published Python functions on TabPy server. + + - [Connecting to TabPy](#connecting-to-tabpy) - [Authentication](#authentication) - [Deploying a Function](#deploying-a-function) - [Predeployed Functions](#predeployed-functions) + * [Principal Component Analysis (PCA)](#principal-component-analysis-pca) + * [Sentiment Analysis](#sentiment-analysis) + * [T-Test](#t-test) - [Providing Schema Metadata](#providing-schema-metadata) - [Querying an Endpoint](#querying-an-endpoint) - [Evaluating Arbitrary Python Scripts](#evaluating-arbitrary-python-scripts) + + ## Connecting to TabPy The tools library uses the notion of connecting to a service to avoid having @@ -39,9 +46,7 @@ has to specify the credentials to use during model deployment with the `set_credentials` call for a client: ```python - client.set_credentials('username', 'P@ssw0rd') - ``` Credentials only need to be set once for all further client operations. @@ -57,13 +62,11 @@ TabPy, see [TabPy Server Configuration Instructions](server-config.md). A persisted endpoint is backed by a Python method. For example: ```python - def add(x,y): import numpy as np return np.add(x, y).tolist() client.deploy('add', add, 'Adds two numbers x and y') - ``` The next example is more complex, using scikit-learn's clustering API: @@ -99,9 +102,7 @@ You can re-deploy a function (for example, after you modified its code) by setti the `override` parameter to `True`: ```python - client.deploy('add', add, 'Adds two numbers x and y', override=True) - ``` Each re-deployment of an endpoint will increment its version number, which is also @@ -135,7 +136,6 @@ executed. In order to get the best performance, we recommended following the methodology outlined in this example. ```python - def LoanDefaultClassifier(Loan_Amount, Loan_Tenure, Monthly_Income, Age): import pandas as pd data=pd.concat([Loan_Amount,Loan_Tenure,Monthly_Income,Age],axis=1) @@ -144,12 +144,12 @@ def LoanDefaultClassifier(Loan_Amount, Loan_Tenure, Monthly_Income, Age): client.deploy('WillItDefault', LoanDefaultClassifier, 'Returns whether a loan application is likely to default.') - ``` You can find a detailed working example with a downloadable sample Tableau workbook and an accompanying Jupyter workbook that walks through model fitting, evaluation -and publishing steps on [our blog](https://www.tableau.com/about/blog/2017/1/building-advanced-analytics-applications-tabpy-64916). +and publishing steps on +[our blog](https://www.tableau.com/about/blog/2017/1/building-advanced-analytics-applications-tabpy-64916). The endpoints that are no longer needed can be removed the following way: @@ -161,34 +161,41 @@ client.remove('WillItDefault') ## Predeployed Functions -To setup models, download the latest version of TabPy and follow the [instructions](server-download.md) -to install and start up your server. Once your server is running, navigate to the -models directory and run setup.py. If your TabPy server is running on the default -config (default.conf), you do not need to specify a config file when launching the -script. If your server is running using a custom config you can specify the config -in the command line like so: +### Deploying Models Shipped With TabPy + +To deploy models shipped with TabPy follow the +[TabPy Installation Instructions](server-install.md) and then +[TabPy Server Configuration Instructions](server-config.md). +Once your server is running execute the following command: ```sh +tabpy-deploy-models +``` -python setup.py custom.conf +If your server is running using a custom config specify the config +in the command line: +```sh +tabpy-deploy-models custom.conf ``` -The setup file will install all of the necessary dependencies `(eg. sklearn, -nltk, textblob, pandas, & numpy)` and deploy all of the prebuilt models -located in `./models/scripts`. For every model that is successfully deployed -a message will be printed to the console: +The command will install all of the necessary dependencies (e.g. `sklearn`, +`nltk`, `textblob`, `pandas`, `numpy`) and deploy all of the prebuilt models. +For every successfully deployed model a message will be printed to the console: ```sh "Successfully deployed PCA" ``` -If you would like to deploy additional models using the deploy script, you can -copy any python file to the `./models/scripts` directory and modify setup.py to -include all necessary packages when installing dependencies, or alternatively install +Use code in [`tabpy/models/scripts`](../tabpy/models/scripts) +as an example of how to create a model and +[`tabpy/models/deploy_models.py`](../tabpy/models/deploy_models.py) +as an example for how to deploy a model. For deployment script include all +necessary packages when installing dependencies or alternatively install all the required dependencies manually. -You can deploy models individually by navigating to models/scripts/ and running +You can deploy models individually by navigating to +[`tabpy/models/scripts`](../tabpy/models/scripts) and running each file in isolation like so: ```sh diff --git a/docs/tabpy-virtualenv.md b/docs/tabpy-virtualenv.md index 3ce31ce0..12a39497 100755 --- a/docs/tabpy-virtualenv.md +++ b/docs/tabpy-virtualenv.md @@ -2,10 +2,7 @@ -To run TabPy in Python virtual environment follow steps -below. - -## Windows Specific Steps +To run TabPy in Python virtual environment follow the steps: 1. Install `virtualenv` package: @@ -13,54 +10,30 @@ below. pip install virtualenv ``` -2. Create virtual environment: - - ```sh - virtualenv - ``` - -3. Activate the environment: - - ```sh - \Scripts\activate - ``` - -4. Run TabPy: +2. Create virtual environment (replace `my-tabpy-env` with + your virtual environment name): ```sh - startup.cmd + virtualenv my-tabpy-env ``` -5. To deactivate virtual environment run: +3. Activate the environment. + 1. For Windows run - ```sh - deactivate - ``` + ```sh + my-tabpy-env\Scripts\activate + ``` -## Linux and Mac Specific Steps + 2. For Linux and Mac run -1. Install `virtualenv` package: - - ```sh - pip install virtualenv - ``` - -2. Create virtual environment: - - ```sh - virtualenv - ``` - -3. Activate the environment: - - ```sh - /bin/activate - ``` + ```sh + my-tabpy-env/bin/activate + ``` 4. Run TabPy: ```sh - ./startup.sh + tabpy ``` 5. To deactivate virtual environment run: diff --git a/setup.py b/setup.py new file mode 100755 index 00000000..40a820cc --- /dev/null +++ b/setup.py @@ -0,0 +1,94 @@ +'''Web server Tableau uses to run Python scripts. + +TabPy (the Tableau Python Server) is an external service implementation +which expands Tableau's capabilities by allowing users to execute Python +scripts and saved functions via Tableau's table calculations. +''' + +DOCLINES = (__doc__ or '').split('\n') + +import os +from setuptools import setup, find_packages + + +def setup_package(): + def read(fname): + return open(os.path.join(os.path.dirname(__file__), fname)).read() + + setup( + name='tabpy', + version=read('VERSION'), + description=DOCLINES[0], + long_description='\n'.join(DOCLINES[1:]) + '\n' + read('CHANGELOG'), + long_description_content_type='text/markdown', + url='https://github.com/tableau/TabPy', + author='Tableau', + author_email='github@tableau.com', + maintainer='Tableau', + maintainer_email='github@tableau.com', + download_url='https://pypi.org/project/tabpy', + project_urls={ + "Bug Tracker": "https://github.com/tableau/TabPy/issues", + "Documentation": "https://tableau.github.io/TabPy/", + "Source Code": "https://github.com/tableau/TabPy", + }, + classifiers=[ + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'Intended Audience :: Science/Research', + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python :: 3.6', + 'Topic :: Scientific/Engineering', + 'Topic :: Scientific/Engineering :: Information Analysis', + 'Operating System :: Microsoft :: Windows', + 'Operating System :: POSIX', + 'Operating System :: Unix', + 'Operating System :: MacOS' + ], + platforms=['Windows', 'Linux', 'Mac OS-X', 'Unix'], + keywords=['tabpy tableau'], + packages=find_packages( + exclude=['docs', 'misc', 'tests']), + package_data={ + 'tabpy': [ + 'VERSION', + 'tabpy_server/state.ini', + 'tabpy_server/static', + 'tabpy_server/common/default.conf' + ] + }, + python_requires='>=3.6', + license='MIT', + # Note: many of these required packages are included in base python + # but are listed here because different linux distros use custom + # python installations. And users can remove packages at any point + install_requires=[ + 'backports_abc', + 'cloudpickle', + 'configparser', + 'decorator', + 'future', + 'genson', + 'jsonschema', + 'mock', + 'numpy', + 'pyopenssl', + 'python-dateutil', + 'requests', + 'singledispatch', + 'six', + 'tornado', + 'urllib3<1.25,>=1.21.1' + ], + entry_points={ + 'console_scripts': [ + 'tabpy=tabpy.tabpy:main', + 'tabpy-deploy-models=tabpy.models.deploy_models:main', + 'tabpy-user-management=tabpy.utils.user_management:main' + ], + } + ) + + +if __name__ == '__main__': + setup_package() diff --git a/startup.cmd b/startup.cmd index a25caa68..48ddf7e7 100755 --- a/startup.cmd +++ b/startup.cmd @@ -1,87 +1,3 @@ @ECHO off -SETLOCAL - - -REM Set environment variables. -SET TABPY_ROOT="%CD%" -SET INSTALL_LOG=%TABPY_ROOT%\tabpy-server\install.log -SET SAVE_PYTHONPATH=%PYTHONPATH% -SET MIN_PY_VER=3.6 -SET DESIRED_PY_VER=3.6.5 - - -ECHO Checking for presence of Python in the system path variable. -SET PYTHON_ERROR=Fatal Error : TabPy startup failed. Check that Python 3.6.5 or higher is installed and is in the system PATH environment variable. -python --version -IF %ERRORLEVEL% NEQ 0 ( - ECHO %PYTHON_ERROR% - SET RET=1 - GOTO:END -) ELSE ( - FOR /F "TOKENS=2" %%a IN ('python --version 2^>^&1') DO ( - IF %%a LSS %MIN_PY_VER% ( - ECHO %PYTHON_ERROR% - SET RET=1 - GOTO:END - ) ELSE IF %%a LSS %DESIRED_PY_VER% ( - ECHO Warning : Python %%a% is not supported. Please upgrade Python to 3.6.5 or higher. - SET RET=1 - ) - ) -) - -REM Install requirements using Python setup tools. -ECHO Installing any missing dependencies... - -CD %TABPY_ROOT%\tabpy-server -ECHO Installing tabpy-server dependencies...>%INSTALL_LOG% -python setup.py install>>%INSTALL_LOG% 2>&1 - -CD %TABPY_ROOT%\tabpy-tools -ECHO: >> %INSTALL_LOG% -ECHO Installing tabpy-tools dependencies...>>%INSTALL_LOG% -python setup.py install>>%INSTALL_LOG% 2>&1 - -CD %TABPY_ROOT% -SET INSTALL_LOG_MESSAGE= Check %INSTALL_LOG% for details. -IF %ERRORLEVEL% NEQ 0 ( - CD %TABPY_ROOT% - ECHO failed - ECHO %INSTALL_LOG_MESSAGE% - SET RET=1 - GOTO:END -) ELSE ( - ECHO success - ECHO %INSTALL_LOG_MESSAGE% -) - - -REM Parse optional CLI arguments: config file -ECHO Parsing parameters... -SET PYTHONPATH=.\tabpy-server;.\tabpy-tools;%PYTHONPATH% -SET STARTUP_CMD=python tabpy-server\tabpy_server\tabpy.py -IF [%1] NEQ [] ( - ECHO Using config file at %1 - SET STARTUP_CMD=%STARTUP_CMD% --config=%1 -) - - -ECHO Starting TabPy server... -ECHO; -%STARTUP_CMD% -IF %ERRORLEVEL% NEQ 0 ( - ECHO Failed to start TabPy server. - SET RET=1 - GOTO:END -) - - -SET RET=%ERRORLEVEL% -GOTO:END - - -:END - SET PYTHONPATH=%SAVE_PYTHONPATH% - CD %TABPY_ROOT% - EXIT /B %RET% - ENDLOCAL +ECHO TabPy is a PIP package now, install it with "pip install tabpy". +ECHO For more information read https://tableau.github.io/TabPy/. diff --git a/startup.sh b/startup.sh index 8724a22f..1057d0b4 100755 --- a/startup.sh +++ b/startup.sh @@ -1,115 +1,3 @@ #!/bin/bash - -min_py_ver=3.6 -desired_py_ver=3.6.5 - -function check_status() { - if [ $? -ne 0 ]; then - echo $1 - exit 1 - fi -} - -function check_python_version() { - python3 --version - check_status $1 - - py_ver=($(python3 --version 2>&1) \| tr ' ' ' ') - if [ "${py_ver[1]}" \< "$min_py_ver" ]; then - echo Fatal Error : $1 - exit 1 - elif [ "${py_ver[1]}" \< "$desired_py_ver" ]; then - echo Warning : Python ${py_ver[1]} is not supported. Please upgrade Python to 3.6.5 or higher. - fi -} - -function install_dependencies() { - # $1 = tabpy_server | tabpy_tools - # $2 = true if install logs are printed to the console, - # false if they are printed to a log file - # $3 = install log file path - if [ "$2" = true ]; then - echo -e "\nInstalling ${1} dependencies..." - python3 setup.py install - elif [ "$2" = false ]; then - echo -e "\nInstalling ${1} dependencies..." >> "${3}" - python3 setup.py install >> "${3}" 2>&1 - else - echo Invalid startup environment. - exit 1 - fi - check_status "Cannot install dependencies." -} - -# Check for Python in PATH -echo Checking for presence of Python in the system path variable. -check_python_version "TabPy startup failed. Check that Python 3.6.5 or higher is installed and is in the system PATH environment variable." - -# Setting local variables -echo Setting TABPY_ROOT to current working directory. -TABPY_ROOT="$PWD" -INSTALL_LOG="${TABPY_ROOT}/tabpy-server/install.log" -echo "" > "${INSTALL_LOG}" -PRINT_INSTALL_LOGS=false - -# Parse CLI parameters -for i in "$@" -do - case $i in - -c=*|--config=*) - CONFIG="${i#*=}" - shift - ;; - --no-startup) - NO_STARTUP=true - shift - ;; - --print-install-logs) - PRINT_INSTALL_LOGS=true - shift - ;; - *) - echo Invalid option: $i - esac -done - -# Check for dependencies, install them if they're not present. -echo Installing TabPy-server requirements. -if [ "$PRINT_INSTALL_LOGS" = false ]; then - echo Read the logs at ${INSTALL_LOG} -fi - -cd "${TABPY_ROOT}/tabpy-server" -install_dependencies "tabpy-server" ${PRINT_INSTALL_LOGS} ${INSTALL_LOG} - -cd "${TABPY_ROOT}/tabpy-tools" -install_dependencies "tabpy-tools" ${PRINT_INSTALL_LOGS} ${INSTALL_LOG} - -cd "${TABPY_ROOT}" -check_status - -if [ ! -z ${CONFIG} ]; then - echo Using the config file at ${TABPY_ROOT}/tabpy-server/$CONFIG. -fi - -# Exit if in a test environent -if [ ! -z ${NO_STARTUP} ]; then - echo Skipping server startup. Exiting successfully. - exit 0 -fi - -# Start TabPy server -echo -echo Starting TabPy server... -SAVE_PYTHONPATH=$PYTHONPATH -export PYTHONPATH="${TABPY_ROOT}/tabpy-server:${TABPY_ROOT}/tabpy-tools:$PYTHONPATH" -if [ -z $CONFIG ]; then - echo Using default parameters. - python3 tabpy-server/tabpy_server/tabpy.py -else - python3 tabpy-server/tabpy_server/tabpy.py --config=$CONFIG -fi - -export PYTHONPATH=$SAVE_PYTHONPATH -check_status -exit 0 +echo TabPy is a PIP package now, install it with "pip install tabpy". +echo For more information read https://tableau.github.io/TabPy/. diff --git a/tabpy-server/setup.py b/tabpy-server/setup.py deleted file mode 100644 index 8da8a3c0..00000000 --- a/tabpy-server/setup.py +++ /dev/null @@ -1,51 +0,0 @@ -try: - from setuptools import setup -except ImportError as err: - print("Missing Python module requirement: setuptools.") - raise err - -from tabpy_server import __version__ - -setup( - name='tabpy-server', - version=__version__, - description='Web server Tableau uses to run Python scripts.', - url='https://github.com/tableau/TabPy', - author='Tableau', - author_email='github@tableau.com', - classifiers=[ - 'Development Status :: 4 - Beta', - 'Intended Audience :: Developers', - 'Programming Language :: Python :: 3.6', - "Topic :: Scientific/Engineering", - "Topic :: Scientific/Engineering :: Information Analysis", - ], - packages=['tabpy_server', - 'tabpy_server.common', - 'tabpy_server.management', - 'tabpy_server.psws', - 'tabpy_server.static'], - package_data={'tabpy_server.static': ['*.*'], - 'tabpy_server': ['startup.*', 'state.ini']}, - license='MIT', - # Note: many of these required packages are included in base python - # but are listed here because different linux distros use custom - # python installations. And users can remove packages at any point - install_requires=[ - 'backports_abc', - 'cloudpickle', - 'configparser', - 'decorator', - 'future', - 'genson', - 'mock', - 'numpy', - 'pyopenssl', - 'python-dateutil', - 'requests', - 'singledispatch', - 'six', - 'tornado', - 'urllib3<1.25,>=1.21.1' - ] -) diff --git a/tabpy-server/tabpy_server/__init__.py b/tabpy-server/tabpy_server/__init__.py deleted file mode 100644 index 7b1e011e..00000000 --- a/tabpy-server/tabpy_server/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -from pathlib import Path - - -def read_version(): - f = None - for path in ['VERSION', '../VERSION', '../../VERSION']: - if Path(path).exists(): - f = path - break - - if f is not None: - with open(f) as h: - return h.read().strip() - else: - return 'dev' - - -__version__ = read_version() diff --git a/tabpy-server/tabpy_server/handlers/__init__.py b/tabpy-server/tabpy_server/handlers/__init__.py deleted file mode 100644 index 7dc480b4..00000000 --- a/tabpy-server/tabpy_server/handlers/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -from tabpy_server.handlers.base_handler import BaseHandler -from tabpy_server.handlers.main_handler import MainHandler -from tabpy_server.handlers.management_handler import ManagementHandler - -from tabpy_server.handlers.endpoint_handler import EndpointHandler -from tabpy_server.handlers.endpoints_handler import EndpointsHandler -from tabpy_server.handlers.evaluation_plane_handler\ - import EvaluationPlaneHandler -from tabpy_server.handlers.query_plane_handler import QueryPlaneHandler -from tabpy_server.handlers.service_info_handler import ServiceInfoHandler -from tabpy_server.handlers.status_handler import StatusHandler -from tabpy_server.handlers.upload_destination_handler\ - import UploadDestinationHandler diff --git a/tabpy-server/tabpy_server/tabpy.py b/tabpy-server/tabpy_server/tabpy.py deleted file mode 100644 index 7df99751..00000000 --- a/tabpy-server/tabpy_server/tabpy.py +++ /dev/null @@ -1,10 +0,0 @@ -from tabpy_server.app.app import TabPyApp - - -def main(): - app = TabPyApp() - app.run() - - -if __name__ == '__main__': - main() diff --git a/tabpy-tools/setup.py b/tabpy-tools/setup.py deleted file mode 100755 index 63a9ae64..00000000 --- a/tabpy-tools/setup.py +++ /dev/null @@ -1,30 +0,0 @@ -try: - from setuptools import setup -except ImportError: - from distutils.core import setup - -from tabpy_tools import __version__ - -setup( - name='tabpy-tools', - version=__version__, - description='Python library of tools to manage TabPy Server.', - url='https://github.com/tableau/TabPy', - author='Tableau', - author_email='github@tableau.com', - # see classifiers at https://pypi.org/pypi?:action=list_classifiers - classifiers=[ - 'Development Status :: 4 - Beta', - 'Intended Audience :: Developers', - 'Programming Language :: Python :: 3.5', - "Topic :: Scientific/Engineering", - "Topic :: Scientific/Engineering :: Information Analysis", - ], - packages=['tabpy_tools'], - license='MIT', - install_requires=[ - 'cloudpickle', - 'requests', - 'genson' - ] -) diff --git a/tabpy-tools/tabpy_tools/__init__.py b/tabpy-tools/tabpy_tools/__init__.py deleted file mode 100755 index 27ad0f0a..00000000 --- a/tabpy-tools/tabpy_tools/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -""" -TabPy client is a Python client to interact with a Tornado-Python-Connector -server process. -""" - -from pathlib import Path - - -def read_version(): - f = None - for path in ['VERSION', '../VERSION', '../../VERSION']: - if Path(path).exists(): - f = path - break - - if f is not None: - with open(f) as h: - return h.read().strip() - else: - return 'dev' - - -__version__ = read_version() diff --git a/models/utils/__init__.py b/tabpy/__init__.py old mode 100644 new mode 100755 similarity index 100% rename from models/utils/__init__.py rename to tabpy/__init__.py diff --git a/tabpy-server/tabpy_server/app/__init__.py b/tabpy/models/__init__.py old mode 100644 new mode 100755 similarity index 100% rename from tabpy-server/tabpy_server/app/__init__.py rename to tabpy/models/__init__.py diff --git a/models/setup.py b/tabpy/models/deploy_models.py similarity index 95% rename from models/setup.py rename to tabpy/models/deploy_models.py index aa337758..c9fe37d1 100644 --- a/models/setup.py +++ b/tabpy/models/deploy_models.py @@ -5,7 +5,7 @@ import runpy import subprocess from pathlib import Path -from utils import setup_utils +from tabpy.models.utils import setup_utils # pip 10.0 introduced a breaking change that moves the location of main try: @@ -22,7 +22,7 @@ def install_dependencies(packages): pip._internal.main(pip_arg) -if __name__ == '__main__': +def main(): install_dependencies(['sklearn', 'pandas', 'numpy', 'textblob', 'nltk', 'scipy']) print('==================================================================') @@ -48,3 +48,6 @@ def install_dependencies(packages): for filename in os.listdir(directory): subprocess.run([py, f'{directory}/{filename}', config_file_path] + auth_args) + +if __name__ == '__main__': + main() diff --git a/models/scripts/PCA.py b/tabpy/models/scripts/PCA.py similarity index 90% rename from models/scripts/PCA.py rename to tabpy/models/scripts/PCA.py index 9c2d68b2..2a2d6b20 100644 --- a/models/scripts/PCA.py +++ b/tabpy/models/scripts/PCA.py @@ -6,8 +6,7 @@ from sklearn.preprocessing import OneHotEncoder import sys from pathlib import Path -sys.path.append(str(Path(__file__).resolve().parent.parent.parent / 'models')) -from utils import setup_utils +from tabpy.models.utils import setup_utils def PCA(component, _arg1, _arg2, *_argN): @@ -59,6 +58,7 @@ def PCA(component, _arg1, _arg2, *_argN): if __name__ == '__main__': - setup_utils.main('PCA', - PCA, - 'Returns the specified principal component') + setup_utils.deploy_model( + 'PCA', + PCA, + 'Returns the specified principal component') diff --git a/models/scripts/SentimentAnalysis.py b/tabpy/models/scripts/SentimentAnalysis.py similarity index 79% rename from models/scripts/SentimentAnalysis.py rename to tabpy/models/scripts/SentimentAnalysis.py index 0b3c3ab6..d3b97e7a 100644 --- a/models/scripts/SentimentAnalysis.py +++ b/tabpy/models/scripts/SentimentAnalysis.py @@ -3,8 +3,12 @@ from nltk.sentiment.vader import SentimentIntensityAnalyzer import sys from pathlib import Path -sys.path.append(str(Path(__file__).resolve().parent.parent.parent / 'models')) -from utils import setup_utils +from tabpy.models.utils import setup_utils + + +import ssl +_ctx = ssl._create_unverified_context +ssl._create_default_https_context = _ctx nltk.download('vader_lexicon') @@ -42,7 +46,8 @@ def SentimentAnalysis(_arg1, library='nltk'): if __name__ == '__main__': - setup_utils.main('Sentiment Analysis', - SentimentAnalysis, - 'Returns a sentiment score between -1 and 1 for ' - 'a given string') + setup_utils.deploy_model( + 'Sentiment Analysis', + SentimentAnalysis, + 'Returns a sentiment score between -1 and 1 for ' + 'a given string') diff --git a/tabpy-server/tabpy_server/common/__init__.py b/tabpy/models/scripts/__init__.py old mode 100644 new mode 100755 similarity index 100% rename from tabpy-server/tabpy_server/common/__init__.py rename to tabpy/models/scripts/__init__.py diff --git a/models/scripts/tTest.py b/tabpy/models/scripts/tTest.py similarity index 85% rename from models/scripts/tTest.py rename to tabpy/models/scripts/tTest.py index d7082698..9bbc0823 100644 --- a/models/scripts/tTest.py +++ b/tabpy/models/scripts/tTest.py @@ -1,8 +1,7 @@ from scipy import stats import sys from pathlib import Path -sys.path.append(str(Path(__file__).resolve().parent.parent.parent / 'models')) -from utils import setup_utils +from tabpy.models.utils import setup_utils def ttest(_arg1, _arg2): @@ -39,6 +38,7 @@ def ttest(_arg1, _arg2): if __name__ == '__main__': - setup_utils.main('ttest', - ttest, - 'Returns the p-value form a t-test') + setup_utils.deploy_model( + 'ttest', + ttest, + 'Returns the p-value form a t-test') diff --git a/tabpy-server/tabpy_server/management/__init__.py b/tabpy/models/utils/__init__.py similarity index 100% rename from tabpy-server/tabpy_server/management/__init__.py rename to tabpy/models/utils/__init__.py diff --git a/models/utils/setup_utils.py b/tabpy/models/utils/setup_utils.py similarity index 79% rename from models/utils/setup_utils.py rename to tabpy/models/utils/setup_utils.py index a5550257..65468153 100644 --- a/models/utils/setup_utils.py +++ b/tabpy/models/utils/setup_utils.py @@ -1,13 +1,16 @@ import configparser -from pathlib import Path import getpass +import os +from pathlib import Path import sys -from tabpy_tools.client import Client +from tabpy.tabpy_tools.client import Client def get_default_config_file_path(): - config_file_path = str(Path(__file__).resolve().parent.parent.parent - / 'tabpy-server/tabpy_server/common/default.conf') + import tabpy + pkg_path = os.path.dirname(tabpy.__file__) + config_file_path = os.path.join( + pkg_path, 'tabpy_server', 'common', 'default.conf') return config_file_path @@ -15,7 +18,11 @@ def parse_config(config_file_path): config = configparser.ConfigParser() config.read(config_file_path) tabpy_config = config['TabPy'] - port = tabpy_config['TABPY_PORT'] + + port = 9004 + if 'TABPY_PORT' in tabpy_config: + port = tabpy_config['TABPY_PORT'] + auth_on = 'TABPY_PWD_FILE' in tabpy_config ssl_on = 'TABPY_TRANSFER_PROTOCOL' in tabpy_config and \ 'TABPY_CERTIFICATE_FILE' in tabpy_config and \ @@ -34,8 +41,8 @@ def get_creds(): return [user, passwd] -def main(funcName, func, funcDescription): - # running from setup.py +def deploy_model(funcName, func, funcDescription): + # running from deploy_models.py if len(sys.argv) > 1: config_file_path = sys.argv[1] else: diff --git a/tabpy/tabpy.py b/tabpy/tabpy.py new file mode 100755 index 00000000..edc3b3ac --- /dev/null +++ b/tabpy/tabpy.py @@ -0,0 +1,31 @@ +import os +from pathlib import Path + + +def read_version(): + ver = 'unknonw' + + import tabpy + pkg_path = os.path.dirname(tabpy.__file__) + ver_file_path = os.path.join(pkg_path, os.path.pardir, 'VERSION') + if Path(ver_file_path).exists(): + with open(ver_file_path) as f: + ver = f.read().strip() + else: + ver = f'Version Unknown, (file {ver_file_path} not found)' + + + return ver + + +__version__ = read_version() + + +def main(): + from tabpy.tabpy_server.app.app import TabPyApp + app = TabPyApp() + app.run() + + +if __name__ == '__main__': + main() diff --git a/tabpy-server/tabpy_server/psws/__init__.py b/tabpy/tabpy_server/__init__.py similarity index 100% rename from tabpy-server/tabpy_server/psws/__init__.py rename to tabpy/tabpy_server/__init__.py diff --git a/tabpy-server/tabpy_server/app/ConfigParameters.py b/tabpy/tabpy_server/app/ConfigParameters.py similarity index 100% rename from tabpy-server/tabpy_server/app/ConfigParameters.py rename to tabpy/tabpy_server/app/ConfigParameters.py diff --git a/tabpy-server/tabpy_server/app/SettingsParameters.py b/tabpy/tabpy_server/app/SettingsParameters.py similarity index 100% rename from tabpy-server/tabpy_server/app/SettingsParameters.py rename to tabpy/tabpy_server/app/SettingsParameters.py diff --git a/tabpy/tabpy_server/app/__init__.py b/tabpy/tabpy_server/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tabpy-server/tabpy_server/app/app.py b/tabpy/tabpy_server/app/app.py similarity index 84% rename from tabpy-server/tabpy_server/app/app.py rename to tabpy/tabpy_server/app/app.py index 3e91d1e5..da9f3cc8 100644 --- a/tabpy-server/tabpy_server/app/app.py +++ b/tabpy/tabpy_server/app/app.py @@ -5,20 +5,23 @@ from logging import config import multiprocessing import os -import tabpy_server -from tabpy_server import __version__ -from tabpy_server.app.ConfigParameters import ConfigParameters -from tabpy_server.app.SettingsParameters import SettingsParameters -from tabpy_server.app.util import parse_pwd_file -from tabpy_server.management.state import TabPyState -from tabpy_server.management.util import _get_state_from_file -from tabpy_server.psws.callbacks import (init_model_evaluator, init_ps_server) -from tabpy_server.psws.python_service import (PythonService, - PythonServiceHandler) -from tabpy_server.handlers import (EndpointHandler, EndpointsHandler, - EvaluationPlaneHandler, QueryPlaneHandler, - ServiceInfoHandler, StatusHandler, - UploadDestinationHandler) +import shutil +import tabpy.tabpy_server +from tabpy.tabpy import __version__ +from tabpy.tabpy_server.app.ConfigParameters import ConfigParameters +from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters +from tabpy.tabpy_server.app.util import parse_pwd_file +from tabpy.tabpy_server.management.state import TabPyState +from tabpy.tabpy_server.management.util import _get_state_from_file +from tabpy.tabpy_server.psws.callbacks\ + import (init_model_evaluator, init_ps_server) +from tabpy.tabpy_server.psws.python_service\ + import (PythonService, PythonServiceHandler) +from tabpy.tabpy_server.handlers\ + import (EndpointHandler, EndpointsHandler, + EvaluationPlaneHandler, QueryPlaneHandler, + ServiceInfoHandler, StatusHandler, + UploadDestinationHandler) import tornado @@ -171,28 +174,37 @@ def set_parameter(settings_key, config_key, default_val=None, check_env_var=False): + key_is_set = False + if config_key is not None and\ parser.has_section('TabPy') and\ parser.has_option('TabPy', config_key): self.settings[settings_key] = parser.get('TabPy', config_key) + key_is_set = True logger.debug( f'Parameter {settings_key} set to ' f'"{self.settings[settings_key]}" ' 'from config file') - elif check_env_var: - self.settings[settings_key] = os.getenv( - config_key, default_val) - logger.debug( - f'Parameter {settings_key} set to ' - f'"{self.settings[settings_key]}" ' - 'from environment variable') - elif default_val is not None: + + if not key_is_set and check_env_var: + val = os.getenv(config_key) + if val is not None: + self.settings[settings_key] = val + key_is_set = True + logger.debug( + f'Parameter {settings_key} set to ' + f'"{self.settings[settings_key]}" ' + 'from environment variable') + + if not key_is_set and default_val is not None: self.settings[settings_key] = default_val + key_is_set = True logger.debug( f'Parameter {settings_key} set to ' f'"{self.settings[settings_key]}" ' 'from default value') - else: + + if not key_is_set: logger.debug( f'Parameter {settings_key} is not set') @@ -213,9 +225,12 @@ def set_parameter(settings_key, 'to evaluate timeout of 30 seconds.') self.settings[SettingsParameters.EvaluateTimeout] = 30 + pkg_path = os.path.dirname(tabpy.__file__) set_parameter(SettingsParameters.UploadDir, ConfigParameters.TABPY_QUERY_OBJECT_PATH, - default_val='/tmp/query_objects', check_env_var=True) + default_val=os.path.join(pkg_path, + 'tmp', 'query_objects'), + check_env_var=True) if not os.path.exists(self.settings[SettingsParameters.UploadDir]): os.makedirs(self.settings[SettingsParameters.UploadDir]) @@ -236,16 +251,23 @@ def set_parameter(settings_key, # last dependence on batch/shell script set_parameter(SettingsParameters.StateFilePath, ConfigParameters.TABPY_STATE_PATH, - default_val='./tabpy-server/tabpy_server', + default_val=os.path.join(pkg_path, 'tabpy_server'), check_env_var=True) self.settings[SettingsParameters.StateFilePath] = os.path.realpath( os.path.normpath( os.path.expanduser( self.settings[SettingsParameters.StateFilePath]))) - state_file_path = self.settings[SettingsParameters.StateFilePath] - logger.info('Loading state from state file ' - f'{os.path.join(state_file_path, "state.ini")}') - tabpy_state = _get_state_from_file(state_file_path) + state_file_dir = self.settings[SettingsParameters.StateFilePath] + state_file_path = os.path.join(state_file_dir, 'state.ini') + if not os.path.isfile(state_file_path): + state_file_template_path = os.path.join( + pkg_path, 'tabpy_server', 'state.ini.template') + logger.debug(f'File {state_file_path} not found, creating from ' + f'template {state_file_template_path}...') + shutil.copy(state_file_template_path, state_file_path) + + logger.info(f'Loading state from state file {state_file_path}') + tabpy_state = _get_state_from_file(state_file_dir) self.tabpy_state = TabPyState( config=tabpy_state, settings=self.settings) @@ -320,7 +342,7 @@ def _validate_transfer_protocol_settings(self): 'an existing file.', os.path.isfile(cert), os.path.isfile(self.settings[SettingsParameters.KeyFile])) - tabpy_server.app.util.validate_cert(cert) + tabpy.tabpy_server.app.util.validate_cert(cert) @staticmethod def _validate_cert_key_state(msg, cert_valid, key_valid): diff --git a/tabpy-server/tabpy_server/app/util.py b/tabpy/tabpy_server/app/util.py similarity index 100% rename from tabpy-server/tabpy_server/app/util.py rename to tabpy/tabpy_server/app/util.py diff --git a/tabpy/tabpy_server/common/__init__.py b/tabpy/tabpy_server/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tabpy-server/tabpy_server/common/default.conf b/tabpy/tabpy_server/common/default.conf similarity index 86% rename from tabpy-server/tabpy_server/common/default.conf rename to tabpy/tabpy_server/common/default.conf index 5dd79d53..3dc12494 100755 --- a/tabpy-server/tabpy_server/common/default.conf +++ b/tabpy/tabpy_server/common/default.conf @@ -1,13 +1,13 @@ [TabPy] -TABPY_QUERY_OBJECT_PATH = /tmp/query_objects -TABPY_PORT = 9004 -TABPY_STATE_PATH = ./tabpy-server/tabpy_server +# TABPY_QUERY_OBJECT_PATH = /tmp/query_objects +# TABPY_PORT = 9004 +# TABPY_STATE_PATH = ./tabpy/tabpy_server # Where static pages live -TABPY_STATIC_PATH = ./tabpy-server/tabpy_server/static +# TABPY_STATIC_PATH = ./tabpy/tabpy_server/static # For how to configure TabPy authentication read -# docs/server-config.md. +# Authentication section in docs/server-config.md. # TABPY_PWD_FILE = /path/to/password/file.txt # To set up secure TabPy uncomment and modify the following lines. diff --git a/tabpy-server/tabpy_server/common/endpoint_file_mgr.py b/tabpy/tabpy_server/common/endpoint_file_mgr.py similarity index 100% rename from tabpy-server/tabpy_server/common/endpoint_file_mgr.py rename to tabpy/tabpy_server/common/endpoint_file_mgr.py diff --git a/tabpy-server/tabpy_server/common/messages.py b/tabpy/tabpy_server/common/messages.py similarity index 100% rename from tabpy-server/tabpy_server/common/messages.py rename to tabpy/tabpy_server/common/messages.py diff --git a/tabpy-server/tabpy_server/common/util.py b/tabpy/tabpy_server/common/util.py similarity index 100% rename from tabpy-server/tabpy_server/common/util.py rename to tabpy/tabpy_server/common/util.py diff --git a/tabpy/tabpy_server/handlers/__init__.py b/tabpy/tabpy_server/handlers/__init__.py new file mode 100644 index 00000000..c73ea4d8 --- /dev/null +++ b/tabpy/tabpy_server/handlers/__init__.py @@ -0,0 +1,13 @@ +from tabpy.tabpy_server.handlers.base_handler import BaseHandler +from tabpy.tabpy_server.handlers.main_handler import MainHandler +from tabpy.tabpy_server.handlers.management_handler import ManagementHandler + +from tabpy.tabpy_server.handlers.endpoint_handler import EndpointHandler +from tabpy.tabpy_server.handlers.endpoints_handler import EndpointsHandler +from tabpy.tabpy_server.handlers.evaluation_plane_handler\ + import EvaluationPlaneHandler +from tabpy.tabpy_server.handlers.query_plane_handler import QueryPlaneHandler +from tabpy.tabpy_server.handlers.service_info_handler import ServiceInfoHandler +from tabpy.tabpy_server.handlers.status_handler import StatusHandler +from tabpy.tabpy_server.handlers.upload_destination_handler\ + import UploadDestinationHandler diff --git a/tabpy-server/tabpy_server/handlers/base_handler.py b/tabpy/tabpy_server/handlers/base_handler.py similarity index 99% rename from tabpy-server/tabpy_server/handlers/base_handler.py rename to tabpy/tabpy_server/handlers/base_handler.py index 3b22f7dc..4ee0934d 100644 --- a/tabpy-server/tabpy_server/handlers/base_handler.py +++ b/tabpy/tabpy_server/handlers/base_handler.py @@ -4,8 +4,8 @@ import json import logging import tornado.web -from tabpy_server.app.SettingsParameters import SettingsParameters -from tabpy_server.handlers.util import hash_password +from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters +from tabpy.tabpy_server.handlers.util import hash_password import uuid diff --git a/tabpy-server/tabpy_server/handlers/endpoint_handler.py b/tabpy/tabpy_server/handlers/endpoint_handler.py similarity index 93% rename from tabpy-server/tabpy_server/handlers/endpoint_handler.py rename to tabpy/tabpy_server/handlers/endpoint_handler.py index cfbf5957..20b6b334 100644 --- a/tabpy-server/tabpy_server/handlers/endpoint_handler.py +++ b/tabpy/tabpy_server/handlers/endpoint_handler.py @@ -6,17 +6,17 @@ at endpoints_handler.py ''' -from tabpy_server.handlers import ManagementHandler +import concurrent import json import logging -import tornado.web -from tornado import gen -from tabpy_server.management.state import get_query_object_path -from tabpy_server.common.util import format_exception -from tabpy_server.handlers.base_handler import STAGING_THREAD -from tabpy_server.psws.callbacks import on_state_change -import concurrent import shutil +from tabpy.tabpy_server.common.util import format_exception +from tabpy.tabpy_server.handlers import ManagementHandler +from tabpy.tabpy_server.handlers.base_handler import STAGING_THREAD +from tabpy.tabpy_server.management.state import get_query_object_path +from tabpy.tabpy_server.psws.callbacks import on_state_change +from tornado import gen +import tornado.web class EndpointHandler(ManagementHandler): diff --git a/tabpy-server/tabpy_server/handlers/endpoints_handler.py b/tabpy/tabpy_server/handlers/endpoints_handler.py similarity index 95% rename from tabpy-server/tabpy_server/handlers/endpoints_handler.py rename to tabpy/tabpy_server/handlers/endpoints_handler.py index 8745e457..bd54311b 100644 --- a/tabpy-server/tabpy_server/handlers/endpoints_handler.py +++ b/tabpy/tabpy_server/handlers/endpoints_handler.py @@ -6,12 +6,12 @@ at endpoint_handler.py ''' -from tabpy_server.handlers import ManagementHandler import json -import tornado.web -from tornado import gen import logging -from tabpy_server.common.util import format_exception +from tabpy.tabpy_server.common.util import format_exception +from tabpy.tabpy_server.handlers import ManagementHandler +from tornado import gen +import tornado.web class EndpointsHandler(ManagementHandler): diff --git a/tabpy-server/tabpy_server/handlers/evaluation_plane_handler.py b/tabpy/tabpy_server/handlers/evaluation_plane_handler.py similarity index 97% rename from tabpy-server/tabpy_server/handlers/evaluation_plane_handler.py rename to tabpy/tabpy_server/handlers/evaluation_plane_handler.py index 2a5686b2..f3d799f6 100644 --- a/tabpy-server/tabpy_server/handlers/evaluation_plane_handler.py +++ b/tabpy/tabpy_server/handlers/evaluation_plane_handler.py @@ -1,7 +1,7 @@ -from tabpy_server.handlers import BaseHandler +from tabpy.tabpy_server.handlers import BaseHandler import json import logging -from tabpy_server.common.util import format_exception +from tabpy.tabpy_server.common.util import format_exception import requests from tornado import gen from datetime import timedelta diff --git a/tabpy-server/tabpy_server/handlers/main_handler.py b/tabpy/tabpy_server/handlers/main_handler.py similarity index 70% rename from tabpy-server/tabpy_server/handlers/main_handler.py rename to tabpy/tabpy_server/handlers/main_handler.py index b43cb8d7..9961da2e 100644 --- a/tabpy-server/tabpy_server/handlers/main_handler.py +++ b/tabpy/tabpy_server/handlers/main_handler.py @@ -1,4 +1,4 @@ -from tabpy_server.handlers import BaseHandler +from tabpy.tabpy_server.handlers import BaseHandler class MainHandler(BaseHandler): diff --git a/tabpy-server/tabpy_server/handlers/management_handler.py b/tabpy/tabpy_server/handlers/management_handler.py similarity index 94% rename from tabpy-server/tabpy_server/handlers/management_handler.py rename to tabpy/tabpy_server/handlers/management_handler.py index 972876f3..805d3e51 100644 --- a/tabpy-server/tabpy_server/handlers/management_handler.py +++ b/tabpy/tabpy_server/handlers/management_handler.py @@ -8,11 +8,11 @@ from tornado import gen -from tabpy_server.app.SettingsParameters import SettingsParameters -from tabpy_server.handlers import MainHandler -from tabpy_server.handlers.base_handler import STAGING_THREAD -from tabpy_server.management.state import get_query_object_path -from tabpy_server.psws.callbacks import on_state_change +from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters +from tabpy.tabpy_server.handlers import MainHandler +from tabpy.tabpy_server.handlers.base_handler import STAGING_THREAD +from tabpy.tabpy_server.management.state import get_query_object_path +from tabpy.tabpy_server.psws.callbacks import on_state_change def copy_from_local(localpath, remotepath, is_dir=False): diff --git a/tabpy-server/tabpy_server/handlers/query_plane_handler.py b/tabpy/tabpy_server/handlers/query_plane_handler.py similarity index 97% rename from tabpy-server/tabpy_server/handlers/query_plane_handler.py rename to tabpy/tabpy_server/handlers/query_plane_handler.py index 1950cbf7..70774626 100644 --- a/tabpy-server/tabpy_server/handlers/query_plane_handler.py +++ b/tabpy/tabpy_server/handlers/query_plane_handler.py @@ -1,12 +1,12 @@ -from tabpy_server.handlers import BaseHandler +from tabpy.tabpy_server.handlers import BaseHandler import logging import time -from tabpy_server.common.messages import ( +from tabpy.tabpy_server.common.messages import ( Query, QuerySuccessful, QueryError, UnknownURI) from hashlib import md5 import uuid import json -from tabpy_server.common.util import format_exception +from tabpy.tabpy_server.common.util import format_exception import urllib import tornado.web from tornado import gen diff --git a/tabpy-server/tabpy_server/handlers/service_info_handler.py b/tabpy/tabpy_server/handlers/service_info_handler.py similarity index 86% rename from tabpy-server/tabpy_server/handlers/service_info_handler.py rename to tabpy/tabpy_server/handlers/service_info_handler.py index 1d1c3059..6341c149 100644 --- a/tabpy-server/tabpy_server/handlers/service_info_handler.py +++ b/tabpy/tabpy_server/handlers/service_info_handler.py @@ -1,6 +1,6 @@ import json -from tabpy_server.app.SettingsParameters import SettingsParameters -from tabpy_server.handlers import ManagementHandler +from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters +from tabpy.tabpy_server.handlers import ManagementHandler class ServiceInfoHandler(ManagementHandler): diff --git a/tabpy-server/tabpy_server/handlers/status_handler.py b/tabpy/tabpy_server/handlers/status_handler.py similarity index 93% rename from tabpy-server/tabpy_server/handlers/status_handler.py rename to tabpy/tabpy_server/handlers/status_handler.py index 60dd9a70..3b2af815 100644 --- a/tabpy-server/tabpy_server/handlers/status_handler.py +++ b/tabpy/tabpy_server/handlers/status_handler.py @@ -1,6 +1,6 @@ import json import logging -from tabpy_server.handlers import BaseHandler +from tabpy.tabpy_server.handlers import BaseHandler class StatusHandler(BaseHandler): diff --git a/tabpy-server/tabpy_server/handlers/upload_destination_handler.py b/tabpy/tabpy_server/handlers/upload_destination_handler.py similarity index 79% rename from tabpy-server/tabpy_server/handlers/upload_destination_handler.py rename to tabpy/tabpy_server/handlers/upload_destination_handler.py index f94ef630..5211b1e6 100644 --- a/tabpy-server/tabpy_server/handlers/upload_destination_handler.py +++ b/tabpy/tabpy_server/handlers/upload_destination_handler.py @@ -1,6 +1,6 @@ import logging -from tabpy_server.app.SettingsParameters import SettingsParameters -from tabpy_server.handlers import ManagementHandler +from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters +from tabpy.tabpy_server.handlers import ManagementHandler import os diff --git a/tabpy-server/tabpy_server/handlers/util.py b/tabpy/tabpy_server/handlers/util.py similarity index 89% rename from tabpy-server/tabpy_server/handlers/util.py rename to tabpy/tabpy_server/handlers/util.py index da7929e0..e835d7fc 100755 --- a/tabpy-server/tabpy_server/handlers/util.py +++ b/tabpy/tabpy_server/handlers/util.py @@ -1,7 +1,7 @@ import base64 import binascii from hashlib import pbkdf2_hmac -from tabpy_server.app.SettingsParameters import SettingsParameters +from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters def hash_password(username, pwd): diff --git a/tabpy/tabpy_server/management/__init__.py b/tabpy/tabpy_server/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tabpy-server/tabpy_server/management/state.py b/tabpy/tabpy_server/management/state.py similarity index 99% rename from tabpy-server/tabpy_server/management/state.py rename to tabpy/tabpy_server/management/state.py index c4ba9bb1..c1353ece 100644 --- a/tabpy-server/tabpy_server/management/state.py +++ b/tabpy/tabpy_server/management/state.py @@ -5,7 +5,7 @@ import json import logging import sys -from tabpy_server.management.util import write_state_config +from tabpy.tabpy_server.management.util import write_state_config from threading import Lock from time import time diff --git a/tabpy-server/tabpy_server/management/util.py b/tabpy/tabpy_server/management/util.py similarity index 93% rename from tabpy-server/tabpy_server/management/util.py rename to tabpy/tabpy_server/management/util.py index 5d93c63f..13d1eae0 100644 --- a/tabpy-server/tabpy_server/management/util.py +++ b/tabpy/tabpy_server/management/util.py @@ -5,8 +5,8 @@ except ImportError: from configparser import ConfigParser as _ConfigParser from datetime import datetime, timedelta, tzinfo -from tabpy_server.app.ConfigParameters import ConfigParameters -from tabpy_server.app.SettingsParameters import SettingsParameters +from tabpy.tabpy_server.app.ConfigParameters import ConfigParameters +from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters def write_state_config(state, settings, logger=logging.getLogger(__name__)): diff --git a/tabpy/tabpy_server/psws/__init__.py b/tabpy/tabpy_server/psws/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tabpy-server/tabpy_server/psws/callbacks.py b/tabpy/tabpy_server/psws/callbacks.py similarity index 93% rename from tabpy-server/tabpy_server/psws/callbacks.py rename to tabpy/tabpy_server/psws/callbacks.py index 903d9998..d6c38b4f 100644 --- a/tabpy-server/tabpy_server/psws/callbacks.py +++ b/tabpy/tabpy_server/psws/callbacks.py @@ -1,12 +1,13 @@ import logging import sys -from tabpy_server.app.SettingsParameters import SettingsParameters -from tabpy_server.common.messages import ( - LoadObject, DeleteObjects, ListObjects, ObjectList) -from tabpy_server.common.endpoint_file_mgr import cleanup_endpoint_files -from tabpy_server.common.util import format_exception -from tabpy_server.management.state import TabPyState, get_query_object_path -from tabpy_server.management import util +from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters +from tabpy.tabpy_server.common.messages\ + import (LoadObject, DeleteObjects, ListObjects, ObjectList) +from tabpy.tabpy_server.common.endpoint_file_mgr import cleanup_endpoint_files +from tabpy.tabpy_server.common.util import format_exception +from tabpy.tabpy_server.management.state\ + import TabPyState, get_query_object_path +from tabpy.tabpy_server.management import util from time import sleep from tornado import gen diff --git a/tabpy-server/tabpy_server/psws/python_service.py b/tabpy/tabpy_server/psws/python_service.py similarity index 98% rename from tabpy-server/tabpy_server/psws/python_service.py rename to tabpy/tabpy_server/psws/python_service.py index f8093623..4a64e810 100644 --- a/tabpy-server/tabpy_server/psws/python_service.py +++ b/tabpy/tabpy_server/psws/python_service.py @@ -3,9 +3,9 @@ import sys -from tabpy_tools.query_object import QueryObject -from tabpy_server.common.util import format_exception -from tabpy_server.common.messages import ( +from tabpy.tabpy_tools.query_object import QueryObject +from tabpy.tabpy_server.common.util import format_exception +from tabpy.tabpy_server.common.messages import ( LoadObject, DeleteObjects, FlushObjects, CountObjects, ListObjects, UnknownMessage, LoadFailed, ObjectsDeleted, ObjectsFlushed, QueryFailed, QuerySuccessful, UnknownURI, DownloadSkipped, LoadInProgress, ObjectCount, diff --git a/tabpy-server/tabpy_server/state.ini b/tabpy/tabpy_server/state.ini.template similarity index 86% rename from tabpy-server/tabpy_server/state.ini rename to tabpy/tabpy_server/state.ini.template index 36b1d2ac..b3828973 100755 --- a/tabpy-server/tabpy_server/state.ini +++ b/tabpy/tabpy_server/state.ini.template @@ -11,4 +11,5 @@ Access-Control-Allow-Methods = [Query Objects Docstrings] [Meta] -Revision Number = 1 \ No newline at end of file +Revision Number = 1 + diff --git a/tabpy-server/tabpy_server/static/index.html b/tabpy/tabpy_server/static/index.html similarity index 100% rename from tabpy-server/tabpy_server/static/index.html rename to tabpy/tabpy_server/static/index.html diff --git a/tabpy-server/tabpy_server/static/tableau.png b/tabpy/tabpy_server/static/tableau.png similarity index 100% rename from tabpy-server/tabpy_server/static/tableau.png rename to tabpy/tabpy_server/static/tableau.png diff --git a/tabpy/tabpy_tools/__init__.py b/tabpy/tabpy_tools/__init__.py new file mode 100755 index 00000000..e69de29b diff --git a/tabpy-tools/tabpy_tools/client.py b/tabpy/tabpy_tools/client.py similarity index 100% rename from tabpy-tools/tabpy_tools/client.py rename to tabpy/tabpy_tools/client.py diff --git a/tabpy-tools/tabpy_tools/custom_query_object.py b/tabpy/tabpy_tools/custom_query_object.py similarity index 100% rename from tabpy-tools/tabpy_tools/custom_query_object.py rename to tabpy/tabpy_tools/custom_query_object.py diff --git a/tabpy-tools/tabpy_tools/query_object.py b/tabpy/tabpy_tools/query_object.py similarity index 100% rename from tabpy-tools/tabpy_tools/query_object.py rename to tabpy/tabpy_tools/query_object.py diff --git a/tabpy-tools/tabpy_tools/rest.py b/tabpy/tabpy_tools/rest.py similarity index 98% rename from tabpy-tools/tabpy_tools/rest.py rename to tabpy/tabpy_tools/rest.py index 7189ff49..44aa6525 100755 --- a/tabpy-tools/tabpy_tools/rest.py +++ b/tabpy/tabpy_tools/rest.py @@ -84,15 +84,18 @@ def GET(self, url, data, timeout=None): self._remove_nones(data) logger.info(f'GET {url} with {data}') + print(f'GET {url} with {data}') response = self.session.get( url, params=data, timeout=timeout, auth=self.auth) + print(f'status_code={response.status_code}') if response.status_code != 200: self.raise_error(response) logger.info(f'response={response.text}') + print(f'response={str(response.text)}') if response.text == '': return dict() diff --git a/tabpy-tools/tabpy_tools/rest_client.py b/tabpy/tabpy_tools/rest_client.py similarity index 100% rename from tabpy-tools/tabpy_tools/rest_client.py rename to tabpy/tabpy_tools/rest_client.py diff --git a/tabpy-tools/tabpy_tools/schema.py b/tabpy/tabpy_tools/schema.py similarity index 98% rename from tabpy-tools/tabpy_tools/schema.py rename to tabpy/tabpy_tools/schema.py index f001d7f0..080d3529 100755 --- a/tabpy-tools/tabpy_tools/schema.py +++ b/tabpy/tabpy_tools/schema.py @@ -1,7 +1,6 @@ import logging import genson as _genson - -from json import validate as _validate +import jsonschema logger = logging.getLogger(__name__) @@ -44,7 +43,7 @@ def _generate_schema_from_example_and_description(input, description): try: # This should not fail unless there are bugs with either genson or # jsonschema. - _validate(input, input_schema) + jsonschema.validate(input, input_schema) except Exception as e: logger.error(f'Internal error validating schema: {str(e)}') raise diff --git a/tabpy/utils/__init__.py b/tabpy/utils/__init__.py new file mode 100755 index 00000000..e69de29b diff --git a/utils/user_management.py b/tabpy/utils/user_management.py similarity index 94% rename from utils/user_management.py rename to tabpy/utils/user_management.py index 45676f93..774af11c 100755 --- a/utils/user_management.py +++ b/tabpy/utils/user_management.py @@ -7,8 +7,8 @@ import os import secrets import sys -from tabpy_server.app.util import parse_pwd_file -from tabpy_server.handlers.util import hash_password +from tabpy.tabpy_server.app.util import parse_pwd_file +from tabpy.tabpy_server.handlers.util import hash_password logger = logging.getLogger(__name__) @@ -132,6 +132,8 @@ def process_command(args, credentials): def main(): + logging.basicConfig(level=logging.DEBUG, format="%(message)s") + parser = build_cli_parser() args = parser.parse_args() if not check_args(args): @@ -150,6 +152,4 @@ def main(): if __name__ == '__main__': - logging.basicConfig(level=logging.DEBUG, format="%(message)s") - main() diff --git a/tests/integration/integ_test_base.py b/tests/integration/integ_test_base.py index 11b0b2a9..13305642 100755 --- a/tests/integration/integ_test_base.py +++ b/tests/integration/integ_test_base.py @@ -1,9 +1,12 @@ +import coverage import http.client import os +from pathlib import Path import platform import shutil import signal import subprocess +import tabpy import tempfile import time import unittest @@ -222,17 +225,19 @@ def setUp(self): # Platform specific - for integration tests we want to engage # startup script with open(self.tmp_dir + '/output.txt', 'w') as outfile: + cmd = ['tabpy', + '--config=' + self.config_file_name] + coverage.process_startup() if platform.system() == 'Windows': self.py = 'python' self.process = subprocess.Popen( - ['startup.cmd', self.config_file_name], + cmd, stdout=outfile, stderr=outfile) else: self.py = 'python3' self.process = subprocess.Popen( - ['./startup.sh', - '--config=' + self.config_file_name], + cmd, preexec_fn=os.setsid, stdout=outfile, stderr=outfile) @@ -272,3 +277,25 @@ def _get_connection(self) -> http.client.HTTPConnection: connection = http.client.HTTPConnection(url) return connection + + def _get_username(self) -> str: + return 'user1' + + def _get_password(self) -> str: + return 'P@ssw0rd' + + def deploy_models(self, username: str, password: str): + repo_dir = os.path.abspath(os.path.dirname(tabpy.__file__)) + path = os.path.join(repo_dir, 'models', 'deploy_models.py') + with open(self.tmp_dir + '/deploy_models_output.txt', 'w') as outfile: + outfile.write( + f'--<< Running {self.py} {path} ' + f'{self._get_config_file_name()} >>--\n') + input_string = f'{username}\n{password}\n' + outfile.write(f'--<< Input = {input_string} >>--') + coverage.process_startup() + p = subprocess.run( + [self.py, path, self._get_config_file_name()], + input=input_string.encode('utf-8'), + stdout=outfile, + stderr=outfile) diff --git a/tests/integration/resources/deploy_and_evaluate_model.conf b/tests/integration/resources/deploy_and_evaluate_model.conf new file mode 100755 index 00000000..375ea47a --- /dev/null +++ b/tests/integration/resources/deploy_and_evaluate_model.conf @@ -0,0 +1,57 @@ +[TabPy] +# TABPY_QUERY_OBJECT_PATH = /tmp/query_objects +TABPY_PORT = 9008 +# TABPY_STATE_PATH = ./tabpy/tabpy_server + +# Where static pages live +# TABPY_STATIC_PATH = ./tabpy/tabpy_server/static + +# For how to configure TabPy authentication read +# Authentication section in docs/server-config.md. +# TABPY_PWD_FILE = /path/to/password/file.txt + +# To set up secure TabPy uncomment and modify the following lines. +# Note only PEM-encoded x509 certificates are supported. +# TABPY_TRANSFER_PROTOCOL = https +# TABPY_CERTIFICATE_FILE = path/to/certificate/file.crt +# TABPY_KEY_FILE = path/to/key/file.key + +# Log additional request details including caller IP, full URL, client +# end user info if provided. +# TABPY_LOG_DETAILS = true + +# Configure how long a custom script provided to the /evaluate method +# will run before throwing a TimeoutError. +# The value should be a float representing the timeout time in seconds. +#TABPY_EVALUATE_TIMEOUT = 30 + +[loggers] +keys=root + +[handlers] +keys=rootHandler,rotatingFileHandler + +[formatters] +keys=rootFormatter + +[logger_root] +level=DEBUG +handlers=rootHandler,rotatingFileHandler +qualname=root +propagete=0 + +[handler_rootHandler] +class=StreamHandler +level=DEBUG +formatter=rootFormatter +args=(sys.stdout,) + +[handler_rotatingFileHandler] +class=handlers.RotatingFileHandler +level=DEBUG +formatter=rootFormatter +args=('tabpy_log.log', 'a', 1000000, 5) + +[formatter_rootFormatter] +format=%(asctime)s [%(levelname)s] (%(filename)s:%(module)s:%(lineno)d): %(message)s +datefmt=%Y-%m-%d,%H:%M:%S diff --git a/tests/integration/test_deploy_and_evaluate_model.py b/tests/integration/test_deploy_and_evaluate_model.py index 57f1bb20..026c5f6b 100644 --- a/tests/integration/test_deploy_and_evaluate_model.py +++ b/tests/integration/test_deploy_and_evaluate_model.py @@ -4,9 +4,19 @@ class TestDeployAndEvaluateModel(integ_test_base.IntegTestBase): + def _get_config_file_name(self) -> str: + return './tests/integration/resources/deploy_and_evaluate_model.conf' + + def _get_port(self) -> str: + return '9008' + def test_deploy_and_evaluate_model(self): - path = str(Path('models', 'setup.py')) - subprocess.call([self.py, path, self._get_config_file_name()]) + # Uncomment the following line to preserve + # test case output and other files (config, state, ect.) + # in system temp folder. + self.set_delete_temp_folder(False) + + self.deploy_models(self._get_username(), self._get_password()) payload = ( '''{ @@ -20,3 +30,4 @@ def test_deploy_and_evaluate_model(self): SentimentAnalysis_eval = conn.getresponse() self.assertEqual(200, SentimentAnalysis_eval.status) SentimentAnalysis_eval.read() + diff --git a/tests/integration/test_deploy_and_evaluate_model_ssl.py b/tests/integration/test_deploy_and_evaluate_model_ssl.py index 669dc724..09a68feb 100755 --- a/tests/integration/test_deploy_and_evaluate_model_ssl.py +++ b/tests/integration/test_deploy_and_evaluate_model_ssl.py @@ -18,8 +18,7 @@ def _get_key_file_name(self) -> str: return './tests/integration/resources/2019_04_24_to_3018_08_25.key' def test_deploy_and_evaluate_model_ssl(self): - path = str(Path('models', 'setup.py')) - subprocess.call([self.py, path, self._get_config_file_name()]) + self.deploy_models(self._get_username(), self._get_password()) payload = ( '''{ diff --git a/tests/integration/test_deploy_model_ssl_off_auth_off.py b/tests/integration/test_deploy_model_ssl_off_auth_off.py index e35d81d8..f5b0749d 100644 --- a/tests/integration/test_deploy_model_ssl_off_auth_off.py +++ b/tests/integration/test_deploy_model_ssl_off_auth_off.py @@ -5,11 +5,11 @@ class TestDeployModelSSLOffAuthOff(integ_test_base.IntegTestBase): def test_deploy_ssl_off_auth_off(self): - models = ['PCA', 'Sentiment%20Analysis', "ttest"] - path = str(Path('models', 'setup.py')) - subprocess.call([self.py, path, self._get_config_file_name()]) + self.deploy_models(self._get_username(), self._get_password()) conn = self._get_connection() + + models = ['PCA', 'Sentiment%20Analysis', "ttest"] for m in models: conn.request("GET", f'/endpoints/{m}') m_request = conn.getresponse() diff --git a/tests/integration/test_deploy_model_ssl_off_auth_on.py b/tests/integration/test_deploy_model_ssl_off_auth_on.py index bb1268eb..0f09bdc6 100644 --- a/tests/integration/test_deploy_model_ssl_off_auth_on.py +++ b/tests/integration/test_deploy_model_ssl_off_auth_on.py @@ -9,10 +9,7 @@ def _get_pwd_file(self) -> str: return './tests/integration/resources/pwdfile.txt' def test_deploy_ssl_off_auth_on(self): - models = ['PCA', 'Sentiment%20Analysis', "ttest"] - path = str(Path('models', 'setup.py')) - p = subprocess.run([self.py, path, self._get_config_file_name()], - input=b'user1\nP@ssw0rd\n') + self.deploy_models(self._get_username(), self._get_password()) headers = { 'Content-Type': "application/json", @@ -24,6 +21,8 @@ def test_deploy_ssl_off_auth_on(self): } conn = self._get_connection() + + models = ['PCA', 'Sentiment%20Analysis', "ttest"] for m in models: conn.request("GET", f'/endpoints/{m}', headers=headers) m_request = conn.getresponse() diff --git a/tests/integration/test_deploy_model_ssl_on_auth_off.py b/tests/integration/test_deploy_model_ssl_on_auth_off.py index fe083849..8c549b47 100644 --- a/tests/integration/test_deploy_model_ssl_on_auth_off.py +++ b/tests/integration/test_deploy_model_ssl_on_auth_off.py @@ -15,9 +15,7 @@ def _get_key_file_name(self) -> str: return './tests/integration/resources/2019_04_24_to_3018_08_25.key' def test_deploy_ssl_on_auth_off(self): - models = ['PCA', 'Sentiment%20Analysis', "ttest"] - path = str(Path('models', 'setup.py')) - subprocess.call([self.py, path, self._get_config_file_name()]) + self.deploy_models(self._get_username(), self._get_password()) session = requests.Session() # Do not verify servers' cert to be signed by trusted CA @@ -25,6 +23,7 @@ def test_deploy_ssl_on_auth_off(self): # Do not warn about insecure request requests.packages.urllib3.disable_warnings() + models = ['PCA', 'Sentiment%20Analysis', "ttest"] for m in models: m_response = session.get(url=f'{self._get_transfer_protocol()}://' f'localhost:9004/endpoints/{m}') diff --git a/tests/integration/test_deploy_model_ssl_on_auth_on.py b/tests/integration/test_deploy_model_ssl_on_auth_on.py index 742abceb..142d6cde 100644 --- a/tests/integration/test_deploy_model_ssl_on_auth_on.py +++ b/tests/integration/test_deploy_model_ssl_on_auth_on.py @@ -2,7 +2,6 @@ import base64 import requests import subprocess -from pathlib import Path class TestDeployModelSSLOnAuthOn(integ_test_base.IntegTestBase): @@ -19,10 +18,12 @@ def _get_pwd_file(self) -> str: return './tests/integration/resources/pwdfile.txt' def test_deploy_ssl_on_auth_on(self): - models = ['PCA', 'Sentiment%20Analysis', "ttest"] - path = str(Path('models', 'setup.py')) - p = subprocess.run([self.py, path, self._get_config_file_name()], - input=b'user1\nP@ssw0rd\n') + # Uncomment the following line to preserve + # test case output and other files (config, state, ect.) + # in system temp folder. + # self.set_delete_temp_folder(False) + + self.deploy_models(self._get_username(), self._get_password()) headers = { 'Content-Type': "application/json", @@ -37,6 +38,7 @@ def test_deploy_ssl_on_auth_on(self): # Do not warn about insecure request requests.packages.urllib3.disable_warnings() + models = ['PCA', 'Sentiment%20Analysis', "ttest"] for m in models: m_response = session.get(url=f'{self._get_transfer_protocol()}://' f'localhost:9004/endpoints/{m}', diff --git a/tests/unit/server_tests/test_config.py b/tests/unit/server_tests/test_config.py index 376cc98a..6d41ec36 100644 --- a/tests/unit/server_tests/test_config.py +++ b/tests/unit/server_tests/test_config.py @@ -2,9 +2,9 @@ import unittest from argparse import Namespace from tempfile import NamedTemporaryFile - -from tabpy_server.app.util import validate_cert -from tabpy_server.app.app import TabPyApp +import tabpy +from tabpy.tabpy_server.app.util import validate_cert +from tabpy.tabpy_server.app.app import TabPyApp from unittest.mock import patch, call @@ -18,41 +18,44 @@ def assert_raises_runtime_error(message, fn, args={}): class TestConfigEnvironmentCalls(unittest.TestCase): - @patch('tabpy_server.app.app.TabPyApp._parse_cli_arguments', + @patch('tabpy.tabpy_server.app.app.TabPyApp._parse_cli_arguments', return_value=Namespace(config=None)) - @patch('tabpy_server.app.app.TabPyState') - @patch('tabpy_server.app.app._get_state_from_file') - @patch('tabpy_server.app.app.PythonServiceHandler') - @patch('tabpy_server.app.app.os.path.exists', return_value=True) - @patch('tabpy_server.app.app.os.path.isfile', return_value=False) - @patch('tabpy_server.app.app.os') - def test_no_config_file(self, mock_os, mock_file_exists, + @patch('tabpy.tabpy_server.app.app.TabPyState') + @patch('tabpy.tabpy_server.app.app._get_state_from_file') + @patch('tabpy.tabpy_server.app.app.PythonServiceHandler') + @patch('tabpy.tabpy_server.app.app.os.path.exists', return_value=True) + @patch('tabpy.tabpy_server.app.app.os') + def test_no_config_file(self, mock_os, mock_path_exists, mock_psws, mock_management_util, mock_tabpy_state, mock_parse_arguments): + pkg_path = os.path.dirname(tabpy.__file__) + obj_path = os.path.join(pkg_path, 'tmp', 'query_objects') + state_path = os.path.join(pkg_path, 'tabpy_server') + + mock_os.getenv.side_effect = [9004, obj_path, state_path] + TabPyApp(None) - getenv_calls = [call('TABPY_PORT', 9004), - call('TABPY_QUERY_OBJECT_PATH', '/tmp/query_objects'), - call('TABPY_STATE_PATH', - './tabpy-server/tabpy_server')] + getenv_calls = [ + call('TABPY_PORT'), + call('TABPY_QUERY_OBJECT_PATH'), + call('TABPY_STATE_PATH')] mock_os.getenv.assert_has_calls(getenv_calls, any_order=True) - self.assertEqual(len(mock_file_exists.mock_calls), 2) self.assertEqual(len(mock_psws.mock_calls), 1) self.assertEqual(len(mock_tabpy_state.mock_calls), 1) self.assertEqual(len(mock_path_exists.mock_calls), 1) self.assertTrue(len(mock_management_util.mock_calls) > 0) mock_os.makedirs.assert_not_called() - @patch('tabpy_server.app.app.TabPyApp._parse_cli_arguments', + @patch('tabpy.tabpy_server.app.app.TabPyApp._parse_cli_arguments', return_value=Namespace(config=None)) - @patch('tabpy_server.app.app.TabPyState') - @patch('tabpy_server.app.app._get_state_from_file') - @patch('tabpy_server.app.app.PythonServiceHandler') - @patch('tabpy_server.app.app.os.path.exists', return_value=False) - @patch('tabpy_server.app.app.os.path.isfile', return_value=False) - @patch('tabpy_server.app.app.os') - def test_no_state_ini_file_or_state_dir(self, mock_os, mock_file_exists, + @patch('tabpy.tabpy_server.app.app.TabPyState') + @patch('tabpy.tabpy_server.app.app._get_state_from_file') + @patch('tabpy.tabpy_server.app.app.PythonServiceHandler') + @patch('tabpy.tabpy_server.app.app.os.path.exists', return_value=False) + @patch('tabpy.tabpy_server.app.app.os') + def test_no_state_ini_file_or_state_dir(self, mock_os, mock_path_exists, mock_psws, mock_management_util, mock_tabpy_state, @@ -69,12 +72,12 @@ def tearDown(self): os.remove(self.config_file.name) self.config_file = None - @patch('tabpy_server.app.app.TabPyApp._parse_cli_arguments') - @patch('tabpy_server.app.app.TabPyState') - @patch('tabpy_server.app.app._get_state_from_file') - @patch('tabpy_server.app.app.PythonServiceHandler') - @patch('tabpy_server.app.app.os.path.exists', return_value=True) - @patch('tabpy_server.app.app.os') + @patch('tabpy.tabpy_server.app.app.TabPyApp._parse_cli_arguments') + @patch('tabpy.tabpy_server.app.app.TabPyState') + @patch('tabpy.tabpy_server.app.app._get_state_from_file') + @patch('tabpy.tabpy_server.app.app.PythonServiceHandler') + @patch('tabpy.tabpy_server.app.app.os.path.exists', return_value=True) + @patch('tabpy.tabpy_server.app.app.os') def test_config_file_present(self, mock_os, mock_path_exists, mock_psws, mock_management_util, mock_tabpy_state, mock_parse_arguments): @@ -91,7 +94,7 @@ def test_config_file_present(self, mock_os, mock_path_exists, mock_os.path.realpath.return_value = 'bar' app = TabPyApp(config_file.name) - getenv_calls = [call('TABPY_PORT', 9004)] + getenv_calls = [call('TABPY_PORT')] mock_os.getenv.assert_has_calls(getenv_calls, any_order=True) self.assertEqual(app.settings['port'], 1234) @@ -105,9 +108,9 @@ def test_config_file_present(self, mock_os, mock_path_exists, self.assertEqual(app.settings['log_request_context'], False) self.assertEqual(app.settings['evaluate_timeout'], 30) - @patch('tabpy_server.app.app.os.path.exists', return_value=True) - @patch('tabpy_server.app.app._get_state_from_file') - @patch('tabpy_server.app.app.TabPyState') + @patch('tabpy.tabpy_server.app.app.os.path.exists', return_value=True) + @patch('tabpy.tabpy_server.app.app._get_state_from_file') + @patch('tabpy.tabpy_server.app.app.TabPyState') def test_custom_evaluate_timeout_valid(self, mock_state, mock_get_state_from_file, mock_path_exists): @@ -120,9 +123,9 @@ def test_custom_evaluate_timeout_valid(self, mock_state, app = TabPyApp(self.config_file.name) self.assertEqual(app.settings['evaluate_timeout'], 1996.0) - @patch('tabpy_server.app.app.os.path.exists', return_value=True) - @patch('tabpy_server.app.app._get_state_from_file') - @patch('tabpy_server.app.app.TabPyState') + @patch('tabpy.tabpy_server.app.app.os.path.exists', return_value=True) + @patch('tabpy.tabpy_server.app.app._get_state_from_file') + @patch('tabpy.tabpy_server.app.app.TabPyState') def test_custom_evaluate_timeout_invalid(self, mock_state, mock_get_state_from_file, mock_path_exists): @@ -197,7 +200,7 @@ def test_https_without_key(self): 'TABPY_KEY_FILE must be set.', TabPyApp, {self.fp.name}) - @patch('tabpy_server.app.app.os.path') + @patch('tabpy.tabpy_server.app.app.os.path') def test_https_cert_and_key_file_not_found(self, mock_path): self.fp.write("[TabPy]\n" "TABPY_TRANSFER_PROTOCOL = https\n" @@ -213,7 +216,7 @@ def test_https_cert_and_key_file_not_found(self, mock_path): 'TABPY_KEY_FILE must point to an existing file.', TabPyApp, {self.fp.name}) - @patch('tabpy_server.app.app.os.path') + @patch('tabpy.tabpy_server.app.app.os.path') def test_https_cert_file_not_found(self, mock_path): self.fp.write("[TabPy]\n" "TABPY_TRANSFER_PROTOCOL = https\n" @@ -229,7 +232,7 @@ def test_https_cert_file_not_found(self, mock_path): 'must point to an existing file.', TabPyApp, {self.fp.name}) - @patch('tabpy_server.app.app.os.path') + @patch('tabpy.tabpy_server.app.app.os.path') def test_https_key_file_not_found(self, mock_path): self.fp.write("[TabPy]\n" "TABPY_TRANSFER_PROTOCOL = https\n" @@ -245,8 +248,8 @@ def test_https_key_file_not_found(self, mock_path): 'must point to an existing file.', TabPyApp, {self.fp.name}) - @patch('tabpy_server.app.app.os.path.isfile', return_value=True) - @patch('tabpy_server.app.util.validate_cert') + @patch('tabpy.tabpy_server.app.app.os.path.isfile', return_value=True) + @patch('tabpy.tabpy_server.app.util.validate_cert') def test_https_success(self, mock_isfile, mock_validate_cert): self.fp.write("[TabPy]\n" "TABPY_TRANSFER_PROTOCOL = HtTpS\n" diff --git a/tests/unit/server_tests/test_endpoint_file_manager.py b/tests/unit/server_tests/test_endpoint_file_manager.py index b87e9e6f..04ddb9cb 100644 --- a/tests/unit/server_tests/test_endpoint_file_manager.py +++ b/tests/unit/server_tests/test_endpoint_file_manager.py @@ -1,5 +1,5 @@ import unittest -from tabpy_server.common.endpoint_file_mgr import _check_endpoint_name +from tabpy.tabpy_server.common.endpoint_file_mgr import _check_endpoint_name class TestEndpointFileManager(unittest.TestCase): diff --git a/tests/unit/server_tests/test_endpoint_handler.py b/tests/unit/server_tests/test_endpoint_handler.py index 2f93e45d..f98135da 100755 --- a/tests/unit/server_tests/test_endpoint_handler.py +++ b/tests/unit/server_tests/test_endpoint_handler.py @@ -3,8 +3,8 @@ import tempfile from argparse import Namespace -from tabpy_server.app.app import TabPyApp -from tabpy_server.handlers.util import hash_password +from tabpy.tabpy_server.app.app import TabPyApp +from tabpy.tabpy_server.handlers.util import hash_password from tornado.testing import AsyncHTTPTestCase from unittest.mock import patch @@ -13,7 +13,7 @@ class TestEndpointHandlerWithAuth(AsyncHTTPTestCase): @classmethod def setUpClass(cls): cls.patcher = patch( - 'tabpy_server.app.app.TabPyApp._parse_cli_arguments', + 'tabpy.tabpy_server.app.app.TabPyApp._parse_cli_arguments', return_value=Namespace( config=None)) cls.patcher.start() diff --git a/tests/unit/server_tests/test_endpoints_handler.py b/tests/unit/server_tests/test_endpoints_handler.py index 096f1cf6..dcb422eb 100755 --- a/tests/unit/server_tests/test_endpoints_handler.py +++ b/tests/unit/server_tests/test_endpoints_handler.py @@ -3,8 +3,8 @@ import tempfile from argparse import Namespace -from tabpy_server.app.app import TabPyApp -from tabpy_server.handlers.util import hash_password +from tabpy.tabpy_server.app.app import TabPyApp +from tabpy.tabpy_server.handlers.util import hash_password from tornado.testing import AsyncHTTPTestCase from unittest.mock import patch @@ -13,7 +13,7 @@ class TestEndpointsHandlerWithAuth(AsyncHTTPTestCase): @classmethod def setUpClass(cls): cls.patcher = patch( - 'tabpy_server.app.app.TabPyApp._parse_cli_arguments', + 'tabpy.tabpy_server.app.app.TabPyApp._parse_cli_arguments', return_value=Namespace( config=None)) cls.patcher.start() diff --git a/tests/unit/server_tests/test_evaluation_plane_handler.py b/tests/unit/server_tests/test_evaluation_plane_handler.py index a02fed5b..45bf6962 100755 --- a/tests/unit/server_tests/test_evaluation_plane_handler.py +++ b/tests/unit/server_tests/test_evaluation_plane_handler.py @@ -3,8 +3,8 @@ import tempfile from argparse import Namespace -from tabpy_server.app.app import TabPyApp -from tabpy_server.handlers.util import hash_password +from tabpy.tabpy_server.app.app import TabPyApp +from tabpy.tabpy_server.handlers.util import hash_password from tornado.testing import AsyncHTTPTestCase from unittest.mock import patch @@ -13,7 +13,7 @@ class TestEvaluationPlainHandlerWithAuth(AsyncHTTPTestCase): @classmethod def setUpClass(cls): cls.patcher = patch( - 'tabpy_server.app.app.TabPyApp._parse_cli_arguments', + 'tabpy.tabpy_server.app.app.TabPyApp._parse_cli_arguments', return_value=Namespace( config=None)) cls.patcher.start() diff --git a/tests/unit/server_tests/test_pwd_file.py b/tests/unit/server_tests/test_pwd_file.py index 6c49194e..596ab9e2 100755 --- a/tests/unit/server_tests/test_pwd_file.py +++ b/tests/unit/server_tests/test_pwd_file.py @@ -2,7 +2,7 @@ import unittest from tempfile import NamedTemporaryFile -from tabpy_server.app.app import TabPyApp +from tabpy.tabpy_server.app.app import TabPyApp class TestPasswordFile(unittest.TestCase): diff --git a/tests/unit/server_tests/test_service_info_handler.py b/tests/unit/server_tests/test_service_info_handler.py index 1abf1399..9f61eeb5 100644 --- a/tests/unit/server_tests/test_service_info_handler.py +++ b/tests/unit/server_tests/test_service_info_handler.py @@ -1,8 +1,8 @@ from argparse import Namespace import json import os -from tabpy_server.app.app import TabPyApp -from tabpy_server.app.SettingsParameters import SettingsParameters +from tabpy.tabpy_server.app.app import TabPyApp +from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters import tempfile from tornado.testing import AsyncHTTPTestCase from unittest.mock import patch @@ -23,7 +23,7 @@ class TestServiceInfoHandlerDefault(AsyncHTTPTestCase): @classmethod def setUpClass(cls): cls.patcher = patch( - 'tabpy_server.app.app.TabPyApp._parse_cli_arguments', + 'tabpy.tabpy_server.app.app.TabPyApp._parse_cli_arguments', return_value=Namespace( config=None)) cls.patcher.start() diff --git a/tests/unit/tools_tests/test_client.py b/tests/unit/tools_tests/test_client.py index 1a7c18dc..670069d0 100644 --- a/tests/unit/tools_tests/test_client.py +++ b/tests/unit/tools_tests/test_client.py @@ -1,7 +1,7 @@ import unittest from unittest.mock import Mock -from tabpy_tools.client import Client +from tabpy.tabpy_tools.client import Client class TestClient(unittest.TestCase): diff --git a/tests/unit/tools_tests/test_rest.py b/tests/unit/tools_tests/test_rest.py index bbd29f1a..530abfee 100644 --- a/tests/unit/tools_tests/test_rest.py +++ b/tests/unit/tools_tests/test_rest.py @@ -1,11 +1,10 @@ import json import requests +from requests.auth import HTTPBasicAuth import sys +from tabpy.tabpy_tools.rest import (RequestsNetworkWrapper, ServiceClient) import unittest from unittest.mock import Mock -from requests.auth import HTTPBasicAuth - -from tabpy_tools.rest import (RequestsNetworkWrapper, ServiceClient) class TestRequestsNetworkWrapper(unittest.TestCase): @@ -20,19 +19,19 @@ def test_init_with_session(self): self.assertIs(session, rnw.session) - def setUp(self): - def mock_response(status_code): - response = Mock(requests.Response()) - response.json.return_value = 'json' - response.status_code = status_code + def mock_response(self, status_code): + response = Mock(requests.Response()) + response.json.return_value = 'json' + response.status_code = status_code - return response + return response + def setUp(self): session = Mock(requests.session()) - session.get.return_value = mock_response(200) - session.post.return_value = mock_response(200) - session.put.return_value = mock_response(200) - session.delete.return_value = mock_response(204) + session.get.return_value = self.mock_response(200) + session.post.return_value = self.mock_response(200) + session.put.return_value = self.mock_response(200) + session.delete.return_value = self.mock_response(204) self.rnw = RequestsNetworkWrapper(session=session) @@ -46,18 +45,18 @@ def test_GET(self): timeout=None, auth=None) - @unittest.expectedFailure def test_GET_InvalidData(self): url = 'abc' data = {'cat'} with self.assertRaises(TypeError): + self.rnw.session.get.return_value = self.mock_response(404) self.rnw.GET(url, data) - @unittest.expectedFailure def test_GET_InvalidURL(self): url = '' data = {'foo': 'bar'} with self.assertRaises(TypeError): + self.rnw.session.get.return_value = self.mock_response(404) self.rnw.GET(url, data) def test_POST(self): @@ -70,11 +69,11 @@ def test_POST(self): timeout=None, auth=None) - @unittest.expectedFailure def test_POST_InvalidURL(self): url = '' data = {'foo': 'bar'} with self.assertRaises(TypeError): + self.rnw.session.post.return_value = self.mock_response(404) self.rnw.POST(url, data) def test_POST_InvalidData(self): diff --git a/tests/unit/tools_tests/test_rest_object.py b/tests/unit/tools_tests/test_rest_object.py index e673d1c6..ec96511e 100644 --- a/tests/unit/tools_tests/test_rest_object.py +++ b/tests/unit/tools_tests/test_rest_object.py @@ -1,7 +1,7 @@ import unittest import sys -from tabpy_tools.rest import RESTObject, RESTProperty, enum +from tabpy.tabpy_tools.rest import RESTObject, RESTProperty, enum class TestRESTObject(unittest.TestCase): diff --git a/tests/unit/tools_tests/test_schema.py b/tests/unit/tools_tests/test_schema.py new file mode 100755 index 00000000..0b7802d2 --- /dev/null +++ b/tests/unit/tools_tests/test_schema.py @@ -0,0 +1,41 @@ +import unittest +import json +from unittest.mock import Mock + +from tabpy.tabpy_tools.schema import generate_schema + + +class TestSchema(unittest.TestCase): + + def test_schema(self): + schema = generate_schema( + input={'x': ['happy', 'sad', 'neutral']}, + input_description={'x': 'text to analyze'}, + output=[.98, -0.99, 0], + output_description='scores for input texts') + expected = { + 'input': { + 'type': 'object', + 'properties': { + 'x': { + 'type': 'array', + 'items': { + 'type': 'string' + }, + 'description': 'text to analyze' + } + }, + 'required': ['x'] + }, + 'sample': { + 'x': ['happy', 'sad', 'neutral'] + }, + 'output': { + 'type': 'array', + 'items': { + 'type': 'number' + }, + 'description': 'scores for input texts' + } + } + self.assertEqual(schema, expected) diff --git a/utils/set_env.cmd b/utils/set_env.cmd deleted file mode 100755 index 45baae54..00000000 --- a/utils/set_env.cmd +++ /dev/null @@ -1,2 +0,0 @@ -@ECHO off -SET PYTHONPATH=%PYTHONPATH%;./tabpy-server;./tabpy-tools; \ No newline at end of file diff --git a/utils/set_env.sh b/utils/set_env.sh deleted file mode 100755 index 4f683fcf..00000000 --- a/utils/set_env.sh +++ /dev/null @@ -1 +0,0 @@ -export PYTHONPATH=./tabpy-server:./tabpy-tools:$PYTHONPATH \ No newline at end of file From 33bed1adf5a88727730804beace7c2062ff2948f Mon Sep 17 00:00:00 2001 From: ogolovatyi Date: Wed, 14 Aug 2019 17:11:56 -0700 Subject: [PATCH 11/28] Merge from master --- tabpy/models/scripts/tTest.py | 7 ++++--- tests/integration/test_deploy_model_ssl_on_auth_on.py | 10 ---------- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/tabpy/models/scripts/tTest.py b/tabpy/models/scripts/tTest.py index d7082698..22118e48 100644 --- a/tabpy/models/scripts/tTest.py +++ b/tabpy/models/scripts/tTest.py @@ -39,6 +39,7 @@ def ttest(_arg1, _arg2): if __name__ == '__main__': - setup_utils.main('ttest', - ttest, - 'Returns the p-value form a t-test') + setup_utils.deploy_model( + 'ttest', + ttest, + 'Returns the p-value form a t-test') diff --git a/tests/integration/test_deploy_model_ssl_on_auth_on.py b/tests/integration/test_deploy_model_ssl_on_auth_on.py index 62a0cc9d..142d6cde 100644 --- a/tests/integration/test_deploy_model_ssl_on_auth_on.py +++ b/tests/integration/test_deploy_model_ssl_on_auth_on.py @@ -18,19 +18,12 @@ def _get_pwd_file(self) -> str: return './tests/integration/resources/pwdfile.txt' def test_deploy_ssl_on_auth_on(self): -<<<<<<< HEAD # Uncomment the following line to preserve # test case output and other files (config, state, ect.) # in system temp folder. # self.set_delete_temp_folder(False) self.deploy_models(self._get_username(), self._get_password()) -======= - models = ['PCA', 'Sentiment%20Analysis', "ttest"] - path = str(Path('models', 'setup.py')) - p = subprocess.run([self.py, path, self._get_config_file_name()], - input=b'user1\nP@ssw0rd\n') ->>>>>>> master headers = { 'Content-Type': "application/json", @@ -45,10 +38,7 @@ def test_deploy_ssl_on_auth_on(self): # Do not warn about insecure request requests.packages.urllib3.disable_warnings() -<<<<<<< HEAD models = ['PCA', 'Sentiment%20Analysis', "ttest"] -======= ->>>>>>> master for m in models: m_response = session.get(url=f'{self._get_transfer_protocol()}://' f'localhost:9004/endpoints/{m}', From a04f21af43fceac96b7d0e10b5ff43d4b6896f9e Mon Sep 17 00:00:00 2001 From: ogolovatyi Date: Wed, 14 Aug 2019 17:13:59 -0700 Subject: [PATCH 12/28] Merge from master --- CHANGELOG | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 91a0c8d5..bd54d1c1 100755 --- a/CHANGELOG +++ b/CHANGELOG @@ -11,16 +11,6 @@ ### Improvements -- Added t-test model -- Fixed models call with /evaluate for HTTPS -- Migrated to Tornado 6 -- Timeout is configurable with TABPY_EVALUATE_TIMEOUT config - file option - -## v0.7 - -### Improvements - - Added t-test model - Fixed models call with /evaluate for HTTPS - Migrated to Tornado 6 From af3a866a8ce83cb233a8c749880e281576a6d1e7 Mon Sep 17 00:00:00 2001 From: ogolovatyi Date: Thu, 15 Aug 2019 09:38:59 -0700 Subject: [PATCH 13/28] Fix code style --- setup.py | 6 ++++-- tabpy/models/deploy_models.py | 1 + tabpy/models/scripts/tTest.py | 3 +-- tabpy/tabpy.py | 1 - tests/integration/test_deploy_and_evaluate_model.py | 1 - 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 40a820cc..1e88870c 100755 --- a/setup.py +++ b/setup.py @@ -5,12 +5,13 @@ scripts and saved functions via Tableau's table calculations. ''' -DOCLINES = (__doc__ or '').split('\n') - import os from setuptools import setup, find_packages +DOCLINES = (__doc__ or '').split('\n') + + def setup_package(): def read(fname): return open(os.path.join(os.path.dirname(__file__), fname)).read() @@ -77,6 +78,7 @@ def read(fname): 'requests', 'singledispatch', 'six', + 'textblob', 'tornado', 'urllib3<1.25,>=1.21.1' ], diff --git a/tabpy/models/deploy_models.py b/tabpy/models/deploy_models.py index c9fe37d1..30dda263 100644 --- a/tabpy/models/deploy_models.py +++ b/tabpy/models/deploy_models.py @@ -49,5 +49,6 @@ def main(): subprocess.run([py, f'{directory}/{filename}', config_file_path] + auth_args) + if __name__ == '__main__': main() diff --git a/tabpy/models/scripts/tTest.py b/tabpy/models/scripts/tTest.py index 22118e48..9bbc0823 100644 --- a/tabpy/models/scripts/tTest.py +++ b/tabpy/models/scripts/tTest.py @@ -1,8 +1,7 @@ from scipy import stats import sys from pathlib import Path -sys.path.append(str(Path(__file__).resolve().parent.parent.parent / 'models')) -from utils import setup_utils +from tabpy.models.utils import setup_utils def ttest(_arg1, _arg2): diff --git a/tabpy/tabpy.py b/tabpy/tabpy.py index edc3b3ac..5d3a4d65 100755 --- a/tabpy/tabpy.py +++ b/tabpy/tabpy.py @@ -14,7 +14,6 @@ def read_version(): else: ver = f'Version Unknown, (file {ver_file_path} not found)' - return ver diff --git a/tests/integration/test_deploy_and_evaluate_model.py b/tests/integration/test_deploy_and_evaluate_model.py index 026c5f6b..3a2e80f5 100644 --- a/tests/integration/test_deploy_and_evaluate_model.py +++ b/tests/integration/test_deploy_and_evaluate_model.py @@ -30,4 +30,3 @@ def test_deploy_and_evaluate_model(self): SentimentAnalysis_eval = conn.getresponse() self.assertEqual(200, SentimentAnalysis_eval.status) SentimentAnalysis_eval.read() - From 2fb487af6c83da41267266e510dde03d03c0752a Mon Sep 17 00:00:00 2001 From: ogolovatyi Date: Thu, 15 Aug 2019 09:55:20 -0700 Subject: [PATCH 14/28] Fix some scrutinizer failures --- tabpy/tabpy.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tabpy/tabpy.py b/tabpy/tabpy.py index 5d3a4d65..b3e095b7 100755 --- a/tabpy/tabpy.py +++ b/tabpy/tabpy.py @@ -1,3 +1,9 @@ +''' +TabPy application. +This file main() function is an entry point for +'tabpy' command. +''' + import os from pathlib import Path From cd0f6a2ae8e54a2c6d61db4298d6bcdf33a03fa6 Mon Sep 17 00:00:00 2001 From: ogolovatyi Date: Thu, 15 Aug 2019 10:12:29 -0700 Subject: [PATCH 15/28] Add .scrutinizer.yml --- .scrutinizer.yml | 24 ++++++++++++++++++++++++ setup.py | 1 - 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100755 .scrutinizer.yml diff --git a/.scrutinizer.yml b/.scrutinizer.yml new file mode 100755 index 00000000..22572143 --- /dev/null +++ b/.scrutinizer.yml @@ -0,0 +1,24 @@ +build: + environment: + python: 3.6 + nodes: + analysis: + project_setup: + override: + - pip install sklearn pandas numpy textblob nltk scipy + tests: + override: + - py-scrutinizer-run + - + command: pylint-run + use_website_config: true + tests: true +checks: + python: + code_rating: true + duplicate_code: true +filter: + excluded_paths: + - '*/test/*' + dependency_paths: + - 'lib/*' diff --git a/setup.py b/setup.py index 1e88870c..0e23cd09 100755 --- a/setup.py +++ b/setup.py @@ -78,7 +78,6 @@ def read(fname): 'requests', 'singledispatch', 'six', - 'textblob', 'tornado', 'urllib3<1.25,>=1.21.1' ], From 98aeacd75fe59e16984d93d36d18607354c5c565 Mon Sep 17 00:00:00 2001 From: ogolovatyi Date: Thu, 15 Aug 2019 11:38:55 -0700 Subject: [PATCH 16/28] Fix scrutinizer test run --- MANIFEST.in | 2 +- VERSION | 2 +- setup.py | 18 ++++++++++++++---- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 0caceefb..b119f809 100755 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,4 +4,4 @@ include \ VERSION \ tabpy/tabpy_server/state.ini \ tabpy/tabpy_server/static/* \ - tabpy/tabpy_server/common/default.conf \ + tabpy/tabpy_server/common/default.conf diff --git a/VERSION b/VERSION index aec258df..ce609caf 100755 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.8 +0.8 \ No newline at end of file diff --git a/setup.py b/setup.py index 0e23cd09..25a577c4 100755 --- a/setup.py +++ b/setup.py @@ -71,8 +71,6 @@ def read(fname): 'future', 'genson', 'jsonschema', - 'mock', - 'numpy', 'pyopenssl', 'python-dateutil', 'requests', @@ -87,8 +85,20 @@ def read(fname): 'tabpy-deploy-models=tabpy.models.deploy_models:main', 'tabpy-user-management=tabpy.utils.user_management:main' ], - } - ) + }, + setup_requires=['pytest-runner'], + tests_require=[ + 'mock', + 'nltk', + 'numpy', + 'pandas', + 'pytest', + 'scipy', + 'sklearn', + 'textblob' + ], + test_suite='pytest' +) if __name__ == '__main__': From da4bed873f03b11149bfd162a2a2be988aea2746 Mon Sep 17 00:00:00 2001 From: ogolovatyi Date: Tue, 20 Aug 2019 08:45:29 -0700 Subject: [PATCH 17/28] Fixing typos in documentation --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6f9702cb..43370b0f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -73,7 +73,7 @@ pytest ### Unit Tests -Unit tests suite can be exectud with the following command: +Unit tests suite can be executed with the following command: ```sh pytest tests/unit @@ -164,7 +164,7 @@ autopep8 -i tabpy-server/server_tests/test_pwd_file.py ## Publishing TabPy Package -Execute the following commands to build and publish new version of +Execute the following commands to build and publish a new version of TabPy package: ```sh From 19f1bc53f618c8c6443e48c5c43aff0fe3f2fc9e Mon Sep 17 00:00:00 2001 From: ogolovatyi Date: Tue, 20 Aug 2019 11:22:13 -0700 Subject: [PATCH 18/28] Fixes based on feedback from jnegara --- docs/server-config.md | 27 ++++++++++++++------------ setup.py | 2 +- tabpy/tabpy_server/common/default.conf | 4 ++-- tabpy/tabpy_tools/rest.py | 3 --- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/docs/server-config.md b/docs/server-config.md index 3d9ebcd4..97d1d354 100755 --- a/docs/server-config.md +++ b/docs/server-config.md @@ -59,12 +59,13 @@ at [`logging.config` documentation page](https://docs.python.org/3.6/library/log - `TABPY_QUERY_OBJECT_PATH` - query objects location. Used with models, see [TabPy Tools documentation](tabpy-tools.md) for details. Default value - `/tmp/query_objects`. -- `TABPY_STATE_PATH` - state location for Tornado web server. Default - value - `tabpy/tabpy_server` subfolder in TabPy package folder. -- `TABPY_STATIC_PATH` - location of static files (index.html page) for - TabPy instance. Default value - `tabpy/tabpy_server/static` subfolder in - TabPy package folder. -- `TABPY_PWD_FILE` - path to password file. Setting up this parameter +- `TABPY_STATE_PATH` - state folder location (absolute path) for Tornado web + server. Default value - `tabpy/tabpy_server` subfolder in TabPy package + folder. +- `TABPY_STATIC_PATH` - absolute path for location of static files (index.html + page) for TabPy instance. Default value - `tabpy/tabpy_server/static` + subfolder in TabPy package folder. +- `TABPY_PWD_FILE` - absolute path to password file. Setting up this parameter makes TabPy require credentials with HTTP(S) requests. More details about authentication can be found in [Authentication](#authentication) section. Default value - not set. @@ -73,10 +74,12 @@ at [`logging.config` documentation page](https://docs.python.org/3.6/library/log `TABPY_CERTIFICATE_FILE` and `TABPY_KEY_FILE`. More details for how to configure TabPy for HTTPS are at [Configuring HTTP vs HTTPS] (#configuring-http-vs-https) section. -- `TABPY_CERTIFICATE_FILE` the certificate file to run TabPy with. Only used - with `TABPY_TRANSFER_PROTOCOL` set to `https`. Default value - not set. -- `TABPY_KEY_FILE` to private key file to run TabPy with. Only used - with `TABPY_TRANSFER_PROTOCOL` set to `https`. Default value - not set. +- `TABPY_CERTIFICATE_FILE` - absolute path to the certificate file to run + TabPy with. Only used with `TABPY_TRANSFER_PROTOCOL` set to `https`. + Default value - not set. +- `TABPY_KEY_FILE` - absolute path to private key file to run TabPy with. + Only used with `TABPY_TRANSFER_PROTOCOL` set to `https`. Default value - + not set. - `TABPY_LOG_DETAILS` - when set to `true` additional call information (caller IP, URL, client info, etc.) is logged. Default value - `false`. - `TABPY_EVALUATE_TIMEOUT` - script evaluation timeout in seconds. Default @@ -103,8 +106,8 @@ settings._ # To set up secure TabPy uncomment and modify the following lines. # Note only PEM-encoded x509 certificates are supported. # TABPY_TRANSFER_PROTOCOL = https -# TABPY_CERTIFICATE_FILE = path/to/certificate/file.crt -# TABPY_KEY_FILE = path/to/key/file.key +# TABPY_CERTIFICATE_FILE = /path/to/certificate/file.crt +# TABPY_KEY_FILE = /path/to/key/file.key # Log additional request details including caller IP, full URL, client # end user info if provided. diff --git a/setup.py b/setup.py index 25a577c4..8676a7d4 100755 --- a/setup.py +++ b/setup.py @@ -98,7 +98,7 @@ def read(fname): 'textblob' ], test_suite='pytest' -) + ) if __name__ == '__main__': diff --git a/tabpy/tabpy_server/common/default.conf b/tabpy/tabpy_server/common/default.conf index 3dc12494..52786491 100755 --- a/tabpy/tabpy_server/common/default.conf +++ b/tabpy/tabpy_server/common/default.conf @@ -13,8 +13,8 @@ # To set up secure TabPy uncomment and modify the following lines. # Note only PEM-encoded x509 certificates are supported. # TABPY_TRANSFER_PROTOCOL = https -# TABPY_CERTIFICATE_FILE = path/to/certificate/file.crt -# TABPY_KEY_FILE = path/to/key/file.key +# TABPY_CERTIFICATE_FILE = /path/to/certificate/file.crt +# TABPY_KEY_FILE = /path/to/key/file.key # Log additional request details including caller IP, full URL, client # end user info if provided. diff --git a/tabpy/tabpy_tools/rest.py b/tabpy/tabpy_tools/rest.py index 44aa6525..7189ff49 100755 --- a/tabpy/tabpy_tools/rest.py +++ b/tabpy/tabpy_tools/rest.py @@ -84,18 +84,15 @@ def GET(self, url, data, timeout=None): self._remove_nones(data) logger.info(f'GET {url} with {data}') - print(f'GET {url} with {data}') response = self.session.get( url, params=data, timeout=timeout, auth=self.auth) - print(f'status_code={response.status_code}') if response.status_code != 200: self.raise_error(response) logger.info(f'response={response.text}') - print(f'response={str(response.text)}') if response.text == '': return dict() From b82ceb4a1b34f6ed4eb1d339f83042b3fb106a2c Mon Sep 17 00:00:00 2001 From: ogolovatyi Date: Wed, 21 Aug 2019 16:33:42 -0700 Subject: [PATCH 19/28] Merge from master --- MANIFEST.in | 5 ----- VERSION | 1 - 2 files changed, 6 deletions(-) delete mode 100755 VERSION diff --git a/MANIFEST.in b/MANIFEST.in index 5071df0b..9eb0d22e 100755 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,12 +1,7 @@ include \ CHANGELOG \ LICENSE \ -<<<<<<< HEAD - VERSION \ - tabpy/tabpy_server/state.ini \ -======= tabpy/VERSION \ tabpy/tabpy_server/state.ini.template \ ->>>>>>> master tabpy/tabpy_server/static/* \ tabpy/tabpy_server/common/default.conf diff --git a/VERSION b/VERSION deleted file mode 100755 index ce609caf..00000000 --- a/VERSION +++ /dev/null @@ -1 +0,0 @@ -0.8 \ No newline at end of file From d411ebc04374fdc44f3d1bf4f7161afec6da5060 Mon Sep 17 00:00:00 2001 From: Jessica Negara Date: Tue, 1 Oct 2019 13:32:24 -0700 Subject: [PATCH 20/28] Added support for environment variables to config file. --- docs/server-config.md | 3 +++ tabpy/tabpy_server/app/app.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/server-config.md b/docs/server-config.md index 97d1d354..1e3fcc97 100755 --- a/docs/server-config.md +++ b/docs/server-config.md @@ -50,6 +50,9 @@ Configuration file consists of settings for TabPy itself and Python logger settings. You should only set parameters if you need different values than the defaults. +Environment variables can be used in the config file. Any instances of +`%(ENV_VAR)s` will be replaced by the value of the environment variable `ENV_VAR`. + TabPy parameters explained below, the logger documentation can be found at [`logging.config` documentation page](https://docs.python.org/3.6/library/logging.config.html). diff --git a/tabpy/tabpy_server/app/app.py b/tabpy/tabpy_server/app/app.py index da9f3cc8..db0824d5 100644 --- a/tabpy/tabpy_server/app/app.py +++ b/tabpy/tabpy_server/app/app.py @@ -160,7 +160,7 @@ def _parse_config(self, config_file): self.python_service = None self.credentials = {} - parser = configparser.ConfigParser() + parser = configparser.ConfigParser(os.environ) if os.path.isfile(config_file): with open(config_file) as f: From 1cb9c737f00b87ab29a7d399b68ccb501a649e41 Mon Sep 17 00:00:00 2001 From: Jessica Negara Date: Tue, 1 Oct 2019 15:22:50 -0700 Subject: [PATCH 21/28] Removed redundant check for environment variables. --- tabpy/tabpy_server/app/app.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tabpy/tabpy_server/app/app.py b/tabpy/tabpy_server/app/app.py index db0824d5..71a0651b 100644 --- a/tabpy/tabpy_server/app/app.py +++ b/tabpy/tabpy_server/app/app.py @@ -186,16 +186,6 @@ def set_parameter(settings_key, f'"{self.settings[settings_key]}" ' 'from config file') - if not key_is_set and check_env_var: - val = os.getenv(config_key) - if val is not None: - self.settings[settings_key] = val - key_is_set = True - logger.debug( - f'Parameter {settings_key} set to ' - f'"{self.settings[settings_key]}" ' - 'from environment variable') - if not key_is_set and default_val is not None: self.settings[settings_key] = default_val key_is_set = True From c004afd02cdc3a32cb7cae2bf2b04ae48c6b52a3 Mon Sep 17 00:00:00 2001 From: Jessica Negara Date: Wed, 2 Oct 2019 11:46:38 -0700 Subject: [PATCH 22/28] Fixed test failures. --- tabpy/tabpy_server/app/app.py | 14 ++++++-------- tests/unit/server_tests/test_config.py | 17 +++++------------ 2 files changed, 11 insertions(+), 20 deletions(-) diff --git a/tabpy/tabpy_server/app/app.py b/tabpy/tabpy_server/app/app.py index 71a0651b..b4b0af21 100644 --- a/tabpy/tabpy_server/app/app.py +++ b/tabpy/tabpy_server/app/app.py @@ -172,8 +172,7 @@ def _parse_config(self, config_file): def set_parameter(settings_key, config_key, - default_val=None, - check_env_var=False): + default_val=None): key_is_set = False if config_key is not None and\ @@ -192,20 +191,21 @@ def set_parameter(settings_key, logger.debug( f'Parameter {settings_key} set to ' f'"{self.settings[settings_key]}" ' - 'from default value') + 'from default value or environment variable') if not key_is_set: logger.debug( f'Parameter {settings_key} is not set') set_parameter(SettingsParameters.Port, ConfigParameters.TABPY_PORT, - default_val=9004, check_env_var=True) + default_val=9004) set_parameter(SettingsParameters.ServerVersion, None, default_val=__version__) set_parameter(SettingsParameters.EvaluateTimeout, ConfigParameters.TABPY_EVALUATE_TIMEOUT, default_val=30) + try: self.settings[SettingsParameters.EvaluateTimeout] = float( self.settings[SettingsParameters.EvaluateTimeout]) @@ -219,8 +219,7 @@ def set_parameter(settings_key, set_parameter(SettingsParameters.UploadDir, ConfigParameters.TABPY_QUERY_OBJECT_PATH, default_val=os.path.join(pkg_path, - 'tmp', 'query_objects'), - check_env_var=True) + 'tmp', 'query_objects')) if not os.path.exists(self.settings[SettingsParameters.UploadDir]): os.makedirs(self.settings[SettingsParameters.UploadDir]) @@ -241,8 +240,7 @@ def set_parameter(settings_key, # last dependence on batch/shell script set_parameter(SettingsParameters.StateFilePath, ConfigParameters.TABPY_STATE_PATH, - default_val=os.path.join(pkg_path, 'tabpy_server'), - check_env_var=True) + default_val=os.path.join(pkg_path, 'tabpy_server')) self.settings[SettingsParameters.StateFilePath] = os.path.realpath( os.path.normpath( os.path.expanduser( diff --git a/tests/unit/server_tests/test_config.py b/tests/unit/server_tests/test_config.py index 6d41ec36..388c95e6 100644 --- a/tests/unit/server_tests/test_config.py +++ b/tests/unit/server_tests/test_config.py @@ -32,16 +32,12 @@ def test_no_config_file(self, mock_os, pkg_path = os.path.dirname(tabpy.__file__) obj_path = os.path.join(pkg_path, 'tmp', 'query_objects') state_path = os.path.join(pkg_path, 'tabpy_server') - - mock_os.getenv.side_effect = [9004, obj_path, state_path] + mock_os.environ = { + 'TABPY_PORT': 9004, 'TABPY_QUERY_OBJECT_PATH': obj_path, + 'TABPY_STATE_PATH': state_path} TabPyApp(None) - getenv_calls = [ - call('TABPY_PORT'), - call('TABPY_QUERY_OBJECT_PATH'), - call('TABPY_STATE_PATH')] - mock_os.getenv.assert_has_calls(getenv_calls, any_order=True) self.assertEqual(len(mock_psws.mock_calls), 1) self.assertEqual(len(mock_tabpy_state.mock_calls), 1) self.assertEqual(len(mock_path_exists.mock_calls), 1) @@ -89,15 +85,12 @@ def test_config_file_present(self, mock_os, mock_path_exists, config_file.close() mock_parse_arguments.return_value = Namespace(config=config_file.name) - - mock_os.getenv.side_effect = [1234] mock_os.path.realpath.return_value = 'bar' + mock_os.environ = {'TABPY_PORT': 1234} app = TabPyApp(config_file.name) - getenv_calls = [call('TABPY_PORT')] - mock_os.getenv.assert_has_calls(getenv_calls, any_order=True) - self.assertEqual(app.settings['port'], 1234) + self.assertEqual(app.settings['port'], '1234') self.assertEqual(app.settings['server_version'], open('VERSION').read().strip()) self.assertEqual(app.settings['upload_dir'], 'foo') From c1934d5eca7361ae6ef2ffb61b0651ffa8143c21 Mon Sep 17 00:00:00 2001 From: Jessica Negara Date: Wed, 2 Oct 2019 13:48:48 -0700 Subject: [PATCH 23/28] Fixed logging messages. --- tabpy/tabpy_server/app/app.py | 4 ++-- tests/unit/server_tests/test_config.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tabpy/tabpy_server/app/app.py b/tabpy/tabpy_server/app/app.py index b4b0af21..7297bc9e 100644 --- a/tabpy/tabpy_server/app/app.py +++ b/tabpy/tabpy_server/app/app.py @@ -183,7 +183,7 @@ def set_parameter(settings_key, logger.debug( f'Parameter {settings_key} set to ' f'"{self.settings[settings_key]}" ' - 'from config file') + 'from config file or environment variable') if not key_is_set and default_val is not None: self.settings[settings_key] = default_val @@ -191,7 +191,7 @@ def set_parameter(settings_key, logger.debug( f'Parameter {settings_key} set to ' f'"{self.settings[settings_key]}" ' - 'from default value or environment variable') + 'from default value') if not key_is_set: logger.debug( diff --git a/tests/unit/server_tests/test_config.py b/tests/unit/server_tests/test_config.py index 388c95e6..2dfad5b6 100644 --- a/tests/unit/server_tests/test_config.py +++ b/tests/unit/server_tests/test_config.py @@ -6,7 +6,7 @@ from tabpy.tabpy_server.app.util import validate_cert from tabpy.tabpy_server.app.app import TabPyApp -from unittest.mock import patch, call +from unittest.mock import patch def assert_raises_runtime_error(message, fn, args={}): @@ -33,7 +33,7 @@ def test_no_config_file(self, mock_os, obj_path = os.path.join(pkg_path, 'tmp', 'query_objects') state_path = os.path.join(pkg_path, 'tabpy_server') mock_os.environ = { - 'TABPY_PORT': 9004, 'TABPY_QUERY_OBJECT_PATH': obj_path, + 'TABPY_PORT': '9004', 'TABPY_QUERY_OBJECT_PATH': obj_path, 'TABPY_STATE_PATH': state_path} TabPyApp(None) @@ -86,7 +86,7 @@ def test_config_file_present(self, mock_os, mock_path_exists, mock_parse_arguments.return_value = Namespace(config=config_file.name) mock_os.path.realpath.return_value = 'bar' - mock_os.environ = {'TABPY_PORT': 1234} + mock_os.environ = {'TABPY_PORT': '1234'} app = TabPyApp(config_file.name) From a68fb18d8b30c08f63dcdddaa93e7f791b6739b9 Mon Sep 17 00:00:00 2001 From: Jessica Negara Date: Wed, 2 Oct 2019 14:12:59 -0700 Subject: [PATCH 24/28] Added unit test for environment variables in config. --- tests/unit/server_tests/test_config.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/unit/server_tests/test_config.py b/tests/unit/server_tests/test_config.py index 2dfad5b6..20c69b80 100644 --- a/tests/unit/server_tests/test_config.py +++ b/tests/unit/server_tests/test_config.py @@ -131,6 +131,21 @@ def test_custom_evaluate_timeout_invalid(self, mock_state, app = TabPyApp(self.config_file.name) self.assertEqual(app.settings['evaluate_timeout'], 30.0) + @patch('tabpy.tabpy_server.app.app.os') + @patch('tabpy.tabpy_server.app.app.os.path.exists', return_value=True) + @patch('tabpy.tabpy_server.app.app._get_state_from_file') + @patch('tabpy.tabpy_server.app.app.TabPyState') + def test_env_variables_in_config(self, mock_state, mock_get_state, + mock_path_exists, mock_os): + mock_os.environ = {'foo': 'baz'} + config_file = self.config_file + config_file.write('[TabPy]\n' + 'TABPY_PORT = %(foo)sbar'.encode()) + config_file.close() + + app = TabPyApp(self.config_file.name) + self.assertEqual(app.settings['port'], 'bazbar') + class TestTransferProtocolValidation(unittest.TestCase): @staticmethod From 9c02b898f6b1cbc32a297660f2f06cab55508624 Mon Sep 17 00:00:00 2001 From: Jessica Date: Wed, 2 Oct 2019 14:24:17 -0700 Subject: [PATCH 25/28] Enable config file to parse environment variables. (#346) * Added support for environment variables to config file. * Removed redundant check for environment variables. * Fixed test failures. * Fixed logging messages. * Added unit test for environment variables in config. --- docs/server-config.md | 3 +++ tabpy/tabpy_server/app/app.py | 26 ++++++-------------- tests/unit/server_tests/test_config.py | 34 ++++++++++++++++---------- 3 files changed, 31 insertions(+), 32 deletions(-) diff --git a/docs/server-config.md b/docs/server-config.md index 7ddcb9cc..39c1ca39 100755 --- a/docs/server-config.md +++ b/docs/server-config.md @@ -50,6 +50,9 @@ Configuration file consists of settings for TabPy itself and Python logger settings. You should only set parameters if you need different values than the defaults. +Environment variables can be used in the config file. Any instances of +`%(ENV_VAR)s` will be replaced by the value of the environment variable `ENV_VAR`. + TabPy parameters explained below, the logger documentation can be found at [`logging.config` documentation page](https://docs.python.org/3.6/library/logging.config.html). diff --git a/tabpy/tabpy_server/app/app.py b/tabpy/tabpy_server/app/app.py index da9f3cc8..7297bc9e 100644 --- a/tabpy/tabpy_server/app/app.py +++ b/tabpy/tabpy_server/app/app.py @@ -160,7 +160,7 @@ def _parse_config(self, config_file): self.python_service = None self.credentials = {} - parser = configparser.ConfigParser() + parser = configparser.ConfigParser(os.environ) if os.path.isfile(config_file): with open(config_file) as f: @@ -172,8 +172,7 @@ def _parse_config(self, config_file): def set_parameter(settings_key, config_key, - default_val=None, - check_env_var=False): + default_val=None): key_is_set = False if config_key is not None and\ @@ -184,17 +183,7 @@ def set_parameter(settings_key, logger.debug( f'Parameter {settings_key} set to ' f'"{self.settings[settings_key]}" ' - 'from config file') - - if not key_is_set and check_env_var: - val = os.getenv(config_key) - if val is not None: - self.settings[settings_key] = val - key_is_set = True - logger.debug( - f'Parameter {settings_key} set to ' - f'"{self.settings[settings_key]}" ' - 'from environment variable') + 'from config file or environment variable') if not key_is_set and default_val is not None: self.settings[settings_key] = default_val @@ -209,13 +198,14 @@ def set_parameter(settings_key, f'Parameter {settings_key} is not set') set_parameter(SettingsParameters.Port, ConfigParameters.TABPY_PORT, - default_val=9004, check_env_var=True) + default_val=9004) set_parameter(SettingsParameters.ServerVersion, None, default_val=__version__) set_parameter(SettingsParameters.EvaluateTimeout, ConfigParameters.TABPY_EVALUATE_TIMEOUT, default_val=30) + try: self.settings[SettingsParameters.EvaluateTimeout] = float( self.settings[SettingsParameters.EvaluateTimeout]) @@ -229,8 +219,7 @@ def set_parameter(settings_key, set_parameter(SettingsParameters.UploadDir, ConfigParameters.TABPY_QUERY_OBJECT_PATH, default_val=os.path.join(pkg_path, - 'tmp', 'query_objects'), - check_env_var=True) + 'tmp', 'query_objects')) if not os.path.exists(self.settings[SettingsParameters.UploadDir]): os.makedirs(self.settings[SettingsParameters.UploadDir]) @@ -251,8 +240,7 @@ def set_parameter(settings_key, # last dependence on batch/shell script set_parameter(SettingsParameters.StateFilePath, ConfigParameters.TABPY_STATE_PATH, - default_val=os.path.join(pkg_path, 'tabpy_server'), - check_env_var=True) + default_val=os.path.join(pkg_path, 'tabpy_server')) self.settings[SettingsParameters.StateFilePath] = os.path.realpath( os.path.normpath( os.path.expanduser( diff --git a/tests/unit/server_tests/test_config.py b/tests/unit/server_tests/test_config.py index 6d26d221..c7270b2f 100644 --- a/tests/unit/server_tests/test_config.py +++ b/tests/unit/server_tests/test_config.py @@ -6,7 +6,7 @@ from tabpy.tabpy_server.app.util import validate_cert from tabpy.tabpy_server.app.app import TabPyApp -from unittest.mock import patch, call +from unittest.mock import patch def assert_raises_runtime_error(message, fn, args={}): @@ -32,16 +32,12 @@ def test_no_config_file(self, mock_os, pkg_path = os.path.dirname(tabpy.__file__) obj_path = os.path.join(pkg_path, 'tmp', 'query_objects') state_path = os.path.join(pkg_path, 'tabpy_server') - - mock_os.getenv.side_effect = [9004, obj_path, state_path] + mock_os.environ = { + 'TABPY_PORT': '9004', 'TABPY_QUERY_OBJECT_PATH': obj_path, + 'TABPY_STATE_PATH': state_path} TabPyApp(None) - getenv_calls = [ - call('TABPY_PORT'), - call('TABPY_QUERY_OBJECT_PATH'), - call('TABPY_STATE_PATH')] - mock_os.getenv.assert_has_calls(getenv_calls, any_order=True) self.assertEqual(len(mock_psws.mock_calls), 1) self.assertEqual(len(mock_tabpy_state.mock_calls), 1) self.assertEqual(len(mock_path_exists.mock_calls), 1) @@ -89,15 +85,12 @@ def test_config_file_present(self, mock_os, mock_path_exists, config_file.close() mock_parse_arguments.return_value = Namespace(config=config_file.name) - - mock_os.getenv.side_effect = [1234] mock_os.path.realpath.return_value = 'bar' + mock_os.environ = {'TABPY_PORT': '1234'} app = TabPyApp(config_file.name) - getenv_calls = [call('TABPY_PORT')] - mock_os.getenv.assert_has_calls(getenv_calls, any_order=True) - self.assertEqual(app.settings['port'], 1234) + self.assertEqual(app.settings['port'], '1234') self.assertEqual(app.settings['server_version'], open('tabpy/VERSION').read().strip()) self.assertEqual(app.settings['upload_dir'], 'foo') @@ -138,6 +131,21 @@ def test_custom_evaluate_timeout_invalid(self, mock_state, app = TabPyApp(self.config_file.name) self.assertEqual(app.settings['evaluate_timeout'], 30.0) + @patch('tabpy.tabpy_server.app.app.os') + @patch('tabpy.tabpy_server.app.app.os.path.exists', return_value=True) + @patch('tabpy.tabpy_server.app.app._get_state_from_file') + @patch('tabpy.tabpy_server.app.app.TabPyState') + def test_env_variables_in_config(self, mock_state, mock_get_state, + mock_path_exists, mock_os): + mock_os.environ = {'foo': 'baz'} + config_file = self.config_file + config_file.write('[TabPy]\n' + 'TABPY_PORT = %(foo)sbar'.encode()) + config_file.close() + + app = TabPyApp(self.config_file.name) + self.assertEqual(app.settings['port'], 'bazbar') + class TestTransferProtocolValidation(unittest.TestCase): @staticmethod From 9e908dbdf87ad4e3a67963f1cc01c419b68ee3c2 Mon Sep 17 00:00:00 2001 From: Jessica Negara Date: Wed, 2 Oct 2019 14:29:00 -0700 Subject: [PATCH 26/28] Updated changelog. --- CHANGELOG | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index 0626a14b..9eb5e61a 100755 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,11 @@ # Changelog +## v0.8.7 + +### Improvements + +- Enabled the use of environment variables in the config file. + ## v0.8.6 ### Fixes From c0fcf1d154d36e0da2913ea5f22ac167db860345 Mon Sep 17 00:00:00 2001 From: Jessica Negara Date: Wed, 2 Oct 2019 14:30:30 -0700 Subject: [PATCH 27/28] Update minor version. --- tabpy/VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tabpy/VERSION b/tabpy/VERSION index 120f5321..35864a97 100755 --- a/tabpy/VERSION +++ b/tabpy/VERSION @@ -1 +1 @@ -0.8.6 \ No newline at end of file +0.8.7 \ No newline at end of file From 8d0b73bd1e24ff1c287035cb64367b03d60eddb1 Mon Sep 17 00:00:00 2001 From: Jessica Negara Date: Thu, 3 Oct 2019 10:22:19 -0700 Subject: [PATCH 28/28] Refactored test_config.py. --- tests/unit/server_tests/test_config.py | 54 ++++++++++++-------------- 1 file changed, 25 insertions(+), 29 deletions(-) diff --git a/tests/unit/server_tests/test_config.py b/tests/unit/server_tests/test_config.py index c7270b2f..2ecffb7e 100644 --- a/tests/unit/server_tests/test_config.py +++ b/tests/unit/server_tests/test_config.py @@ -9,14 +9,6 @@ from unittest.mock import patch -def assert_raises_runtime_error(message, fn, args={}): - try: - fn(*args) - assert False - except RuntimeError as err: - assert err.args[0] == message - - class TestConfigEnvironmentCalls(unittest.TestCase): @patch('tabpy.tabpy_server.app.app.TabPyApp._parse_cli_arguments', return_value=Namespace(config=None)) @@ -148,6 +140,11 @@ def test_env_variables_in_config(self, mock_state, mock_get_state, class TestTransferProtocolValidation(unittest.TestCase): + def assertTabPyAppRaisesRuntimeError(self, expected_message): + with self.assertRaises(RuntimeError) as err: + TabPyApp(self.fp.name) + self.assertEqual(err.exception.args[0], expected_message) + @staticmethod def mock_isfile(target_file, existing_files): if target_file in existing_files: @@ -182,10 +179,9 @@ def test_https_without_cert_and_key(self): "TABPY_TRANSFER_PROTOCOL = https") self.fp.close() - assert_raises_runtime_error( - 'Error using HTTPS: The parameter(s) TABPY_CERTIFICATE_FILE ' - 'and TABPY_KEY_FILE must be set.', - TabPyApp, {self.fp.name}) + self.assertTabPyAppRaisesRuntimeError('Error using HTTPS: The paramete' + 'r(s) TABPY_CERTIFICATE_FILE and' + ' TABPY_KEY_FILE must be set.') def test_https_without_cert(self): self.fp.write( @@ -194,9 +190,9 @@ def test_https_without_cert(self): "TABPY_KEY_FILE = foo") self.fp.close() - assert_raises_runtime_error('Error using HTTPS: The parameter(s) ' - 'TABPY_CERTIFICATE_FILE must be set.', - TabPyApp, {self.fp.name}) + self.assertTabPyAppRaisesRuntimeError( + 'Error using HTTPS: The parameter(s) TABPY_CERTIFICATE_FILE must ' + 'be set.') def test_https_without_key(self): self.fp.write("[TabPy]\n" @@ -204,9 +200,8 @@ def test_https_without_key(self): "TABPY_CERTIFICATE_FILE = foo") self.fp.close() - assert_raises_runtime_error('Error using HTTPS: The parameter(s) ' - 'TABPY_KEY_FILE must be set.', - TabPyApp, {self.fp.name}) + self.assertTabPyAppRaisesRuntimeError( + 'Error using HTTPS: The parameter(s) TABPY_KEY_FILE must be set.') @patch('tabpy.tabpy_server.app.app.os.path') def test_https_cert_and_key_file_not_found(self, mock_path): @@ -219,10 +214,9 @@ def test_https_cert_and_key_file_not_found(self, mock_path): mock_path.isfile.side_effect = lambda x: self.mock_isfile( x, {self.fp.name}) - assert_raises_runtime_error( + self.assertTabPyAppRaisesRuntimeError( 'Error using HTTPS: The parameter(s) TABPY_CERTIFICATE_FILE and ' - 'TABPY_KEY_FILE must point to an existing file.', - TabPyApp, {self.fp.name}) + 'TABPY_KEY_FILE must point to an existing file.') @patch('tabpy.tabpy_server.app.app.os.path') def test_https_cert_file_not_found(self, mock_path): @@ -235,10 +229,9 @@ def test_https_cert_file_not_found(self, mock_path): mock_path.isfile.side_effect = lambda x: self.mock_isfile( x, {self.fp.name, 'bar'}) - assert_raises_runtime_error( + self.assertTabPyAppRaisesRuntimeError( 'Error using HTTPS: The parameter(s) TABPY_CERTIFICATE_FILE ' - 'must point to an existing file.', - TabPyApp, {self.fp.name}) + 'must point to an existing file.') @patch('tabpy.tabpy_server.app.app.os.path') def test_https_key_file_not_found(self, mock_path): @@ -251,10 +244,9 @@ def test_https_key_file_not_found(self, mock_path): mock_path.isfile.side_effect = lambda x: self.mock_isfile( x, {self.fp.name, 'foo'}) - assert_raises_runtime_error( + self.assertTabPyAppRaisesRuntimeError( 'Error using HTTPS: The parameter(s) TABPY_KEY_FILE ' - 'must point to an existing file.', - TabPyApp, {self.fp.name}) + 'must point to an existing file.') @patch('tabpy.tabpy_server.app.app.os.path.isfile', return_value=True) @patch('tabpy.tabpy_server.app.util.validate_cert') @@ -273,6 +265,10 @@ def test_https_success(self, mock_isfile, mock_validate_cert): class TestCertificateValidation(unittest.TestCase): + def assertValidateCertRaisesRuntimeError(self, expected_message, path): + with self.assertRaises(RuntimeError) as err: + validate_cert(path) + self.assertEqual(err.exception.args[0], expected_message) def __init__(self, *args, **kwargs): super(TestCertificateValidation, self).__init__(*args, **kwargs) @@ -283,13 +279,13 @@ def test_expired_cert(self): path = os.path.join(self.resources_path, 'expired.crt') message = ('Error using HTTPS: The certificate provided expired ' 'on 2018-08-18 19:47:18.') - assert_raises_runtime_error(message, validate_cert, {path}) + self.assertValidateCertRaisesRuntimeError(message, path) def test_future_cert(self): path = os.path.join(self.resources_path, 'future.crt') message = ('Error using HTTPS: The certificate provided is not valid ' 'until 3001-01-01 00:00:00.') - assert_raises_runtime_error(message, validate_cert, {path}) + self.assertValidateCertRaisesRuntimeError(message, path) def test_valid_cert(self): path = os.path.join(self.resources_path, 'valid.crt')