Dev improvements (tableau#384)

* Fix flake8 warnings

* Merge from master

* Fix pycodestyle

* Fix more flake8 warnings

* Fix tests to pass again

* Create test_coveralls_codestyle.yml (tableau#382)

* Use github actions

.github/workflows/test_coveralls_codestyle.yml

@@ -0,0 +1,108 @@
+name: CI
+
+on: [push, pull_request]
+
+jobs:
+  ubuntu-build:
+    name: ${{ matrix.python-version }} on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        python-version: [3.6, 3.7]
+        os: [ubuntu-latest]
+
+    steps:
+    - uses: actions/checkout@v1
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+
+    - name: Lint with flake8
+      run: |
+        pip install flake8
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+        
+    - name: Python Style Checker
+      uses: andymckay/[email protected]
+
+    - name: Test with pytest
+      run: |
+        pip install pytest pytest-cov coveralls coverage==4.5.4
+        pytest tests/unit --cov=tabpy --cov-append
+        pytest tests/integration --cov=tabpy --cov-append
+        coveralls
+      env:
+        COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
+
+    - name: Markdownlint
+      uses: nosborn/[email protected]
+      with:
+        files: .
+
+  windows-build:
+    name: ${{ matrix.python-version }} on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        python-version: [3.7]
+        os: [windows-latest]
+
+    steps:
+    - uses: actions/checkout@v1
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+
+    - name: Test with pytest
+      run: |
+        pip install pytest pytest-cov coveralls
+        pytest tests/unit
+        pytest tests/integration
+
+  mac-build:
+    name: ${{ matrix.python-version }} on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        python-version: [3.7]
+        os: [macos-latest]
+
+    steps:
+    - uses: actions/checkout@v1
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+
+    - name: Test with pytest
+      run: |
+        pip install pytest pytest-cov coveralls
+        pytest tests/unit
+        pytest tests/integration
+        

.travis.yml

@@ -2,6 +2,7 @@ os: linux
 language: python
 python: 3.6
 install:
+    - python -m pip install --upgrade pip
     - pip install pytest pytest-cov coveralls
     - npm install -g markdownlint-cli
 script:

CONTRIBUTING.md

@@ -31,7 +31,7 @@ and run it locally.
 These are prerequisites for an environment required for a contributor to
 be able to work on TabPy changes:
 
-- Python 3.6.5:
+- Python 3.6 or 3.7:
   - To see which version of Python you have installed, run `python --version`.
 - git
 - TabPy repo:
@@ -57,16 +57,10 @@ be able to work on TabPy changes:
     cd TabPy
     ```
 
-4. Register TabPy repo as a pip package:
-
-    ```sh
-    pip install -e .
-    ```
-
-5. Install all dependencies:
+4. Install all dependencies:
 
    ```sh
-   python setup.py install
+   pip install -r requirements.txt
    ```
 
 ## Tests

README.md

@@ -7,6 +7,7 @@
 [![Scrutinizer Code Quality](https://scrutinizer-ci.com/g/tableau/TabPy/badges/quality-score.png?b=master)](https://scrutinizer-ci.com/g/tableau/TabPy/?branch=master)
 
 [![Python 3.6](https://img.shields.io/badge/python-3.6-blue.svg)](https://www.python.org/downloads/release/python-360/)
+[![Python 3.7](https://img.shields.io/badge/python-3.7-blue.svg)](https://www.python.org/downloads/release/python-370/)
 
 ![Release](https://img.shields.io/github/release/tableau/TabPy.svg)
 

requirements.txt

@@ -0,0 +1,3 @@
+# installs dependencies from ./setup.py, and the package itself,
+# in editable mode
+-e .

setup.cfg

@@ -0,0 +1,2 @@
+[pycodestyle]
+max-line-length = 88

tabpy/models/deploy_models.py

@@ -9,11 +9,8 @@
 
 def install_dependencies(packages):
     pip_arg = ["install"] + packages + ["--no-cache-dir"]
-    if hasattr(pip, "main"):
-        pip.main(pip_arg)
-    else:
-        from pip._internal import main
-        pip._internal.main(pip_arg)
+    from pip._internal import main
+    pip._internal.main.main(pip_arg)
 
 
 def main():

tabpy/tabpy_server/handlers/base_handler.py

@@ -311,8 +311,7 @@ def _get_credentials(self, method) -> bool:
         # No known methods were found
         self.logger.log(
             logging.CRITICAL,
-            f'Unknown authentication method(s) "{method}" are configured '
-            f'for API "{api_version}"',
+            f'Unknown authentication method(s) "{method}" are configured ',
         )
         return False
 
@@ -368,8 +367,7 @@ def _validate_credentials(self, method) -> bool:
         # No known methods were found
         self.logger.log(
             logging.CRITICAL,
-            f'Unknown authentication method(s) "{method}" are configured '
-            f'for API "{api_version}"',
+            f'Unknown authentication method(s) "{method}" are configured ',
         )
         return False
 

tabpy/tabpy_server/handlers/endpoints_handler.py

@@ -11,7 +11,6 @@
 from tabpy.tabpy_server.common.util import format_exception
 from tabpy.tabpy_server.handlers import ManagementHandler
 from tornado import gen
-import tornado.web
 
 
 class EndpointsHandler(ManagementHandler):

tabpy/tabpy_server/handlers/evaluation_plane_handler.py

@@ -127,10 +127,16 @@ def _call_subprocess(self, function_to_evaluate, arguments):
         # Exec does not run the function, so it does not block.
         exec(function_to_evaluate, globals())
 
+        # 'noqa' comments below tell flake8 to ignore undefined _user_script
+        # name - the name is actually defined with user script being wrapped
+        # in _user_script function (constructed as a striong) and then executed
+        # with exec() call above.
         if arguments is None:
-            future = self.executor.submit(_user_script, restricted_tabpy)
+            future = self.executor.submit(_user_script,  # noqa: F821
+                                          restricted_tabpy)
         else:
-            future = self.executor.submit(_user_script, restricted_tabpy, **arguments)
+            future = self.executor.submit(_user_script,  # noqa: F821
+                                          restricted_tabpy, **arguments)
 
         ret = yield gen.with_timeout(timedelta(seconds=self.eval_timeout), future)
         raise gen.Return(ret)

tabpy/tabpy_server/management/util.py

@@ -5,7 +5,6 @@
     from ConfigParser import ConfigParser as _ConfigParser
 except ImportError:
     from configparser import ConfigParser as _ConfigParser
-from datetime import datetime, timedelta, tzinfo
 from tabpy.tabpy_server.app.ConfigParameters import ConfigParameters
 from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters
 

tabpy/tabpy_server/psws/callbacks.py

@@ -1,5 +1,4 @@
 import logging
-import sys
 from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters
 from tabpy.tabpy_server.common.messages import (
     LoadObject,

tabpy/tabpy_tools/client.py

@@ -1,10 +1,11 @@
+import copy
 from re import compile
 import time
 import requests
 
 from .rest import RequestsNetworkWrapper, ServiceClient
 
-from .rest_client import RESTServiceClient, Endpoint, AliasEndpoint
+from .rest_client import RESTServiceClient, Endpoint
 
 from .custom_query_object import CustomQueryObject
 import os
@@ -313,7 +314,7 @@ def _gen_endpoint(self, name, obj, description, version=1, schema=[]):
             "methods": endpoint_object.get_methods(),
             "required_files": [],
             "required_packages": [],
-            "schema": schema,
+            "schema": copy.copy(schema),
         }
 
     def _upload_endpoint(self, obj):

tabpy/tabpy_tools/rest.py

@@ -101,7 +101,7 @@ def POST(self, url, data, timeout=None):
         response = self.session.post(
             url,
             data=data,
-            headers={"content-type": "application/json",},
+            headers={"content-type": "application/json"},
             timeout=timeout,
             auth=self.auth,
         )
@@ -121,7 +121,7 @@ def PUT(self, url, data, timeout=None):
         response = self.session.put(
             url,
             data=data,
-            headers={"content-type": "application/json",},
+            headers={"content-type": "application/json"},
             timeout=timeout,
             auth=self.auth,
         )
@@ -418,6 +418,6 @@ def __new__(cls, value):
 
         return super(enum, cls).__new__(cls, value)
 
-    enum = type("Enum", (enum_type,), {"values": values, "__new__": __new__,})
+    enum = type("Enum", (enum_type,), {"values": values, "__new__": __new__})
 
     return enum

tabpy/tabpy_tools/rest_client.py

@@ -50,7 +50,7 @@ class Endpoint(RESTObject):
 
     def __new__(cls, **kwargs):
         """Dispatch to the appropriate class."""
-        cls = {"alias": AliasEndpoint, "model": ModelEndpoint,}[kwargs["type"]]
+        cls = {"alias": AliasEndpoint, "model": ModelEndpoint}[kwargs["type"]]
 
         """return object.__new__(cls, **kwargs)"""
         """ modified for Python 3"""

tests/integration/integ_test_base.py

@@ -1,7 +1,6 @@
 import coverage
 import http.client
 import os
-from pathlib import Path
 import platform
 import shutil
 import signal
@@ -292,7 +291,7 @@ def deploy_models(self, username: str, password: str):
             input_string = f"{username}\n{password}\n"
             outfile.write(f"--<< Input = {input_string} >>--")
             coverage.process_startup()
-            p = subprocess.run(
+            subprocess.run(
                 [self.py, path, self._get_config_file_name()],
                 input=input_string.encode("utf-8"),
                 stdout=outfile,

tests/integration/test_custom_evaluate_timeout.py

@@ -15,8 +15,8 @@ def test_custom_evaluate_timeout_with_script(self):
             """
         headers = {
             "Content-Type": "application/json",
-            "TabPy-Client": "Integration test for testing custom evaluate timeouts with "
-            "scripts.",
+            "TabPy-Client": "Integration test for testing custom evaluate timeouts "
+            "with scripts.",
         }
 
         conn = self._get_connection()

tests/integration/test_deploy_and_evaluate_model.py

@@ -1,6 +1,4 @@
 import integ_test_base
-import subprocess
-from pathlib import Path
 
 
 class TestDeployAndEvaluateModel(integ_test_base.IntegTestBase):

tests/integration/test_deploy_and_evaluate_model_ssl.py

@@ -1,7 +1,5 @@
 import integ_test_base
 import requests
-import subprocess
-from pathlib import Path
 
 
 class TestDeployAndEvaluateModelSSL(integ_test_base.IntegTestBase):

tests/integration/test_deploy_model_ssl_off_auth_off.py

@@ -1,6 +1,4 @@
 import integ_test_base
-import subprocess
-from pathlib import Path
 
 
 class TestDeployModelSSLOffAuthOff(integ_test_base.IntegTestBase):

tests/integration/test_deploy_model_ssl_off_auth_on.py

@@ -1,7 +1,5 @@
 import integ_test_base
 import base64
-import subprocess
-from pathlib import Path
 
 
 class TestDeployModelSSLOffAuthOn(integ_test_base.IntegTestBase):

tests/integration/test_deploy_model_ssl_on_auth_off.py

@@ -1,7 +1,5 @@
 import integ_test_base
 import requests
-import subprocess
-from pathlib import Path
 
 
 class TestDeployModelSSLOnAuthOff(integ_test_base.IntegTestBase):

tests/integration/test_deploy_model_ssl_on_auth_on.py

@@ -1,7 +1,6 @@
 import integ_test_base
 import base64
 import requests
-import subprocess
 
 
 class TestDeployModelSSLOnAuthOn(integ_test_base.IntegTestBase):

tests/unit/server_tests/test_service_info_handler.py

@@ -120,7 +120,7 @@ def test_given_tabpy_server_with_auth_expect_correct_info_response(self):
         self.assertTrue("features" in v1)
         features = v1["features"]
         self.assertDictEqual(
-            {"authentication": {"methods": {"basic-auth": {}}, "required": True,}},
+            {"authentication": {"methods": {"basic-auth": {}}, "required": True}},
             features,
         )
 

tests/unit/tools_tests/test_rest.py

@@ -1,7 +1,6 @@
 import json
 import requests
 from requests.auth import HTTPBasicAuth
-import sys
 from tabpy.tabpy_tools.rest import RequestsNetworkWrapper, ServiceClient
 import unittest
 from unittest.mock import Mock

tests/unit/tools_tests/test_schema.py

@@ -1,6 +1,4 @@
 import unittest
-import json
-from unittest.mock import Mock
 
 from tabpy.tabpy_tools.schema import generate_schema