Skip to content

Commit

Permalink
Remove Python 3.5 support/apply black+isort+pre-commit
Browse files Browse the repository at this point in the history
Add Python 3.8 support
  • Loading branch information
MartinThoma committed Sep 7, 2020
1 parent f2983ec commit 425eaaf
Show file tree
Hide file tree
Showing 53 changed files with 203 additions and 147 deletions.
9 changes: 6 additions & 3 deletions .github/workflows/static-code-analysis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: [3.7]
python-version: [3.8]

steps:
- uses: actions/checkout@v2
Expand All @@ -29,5 +29,8 @@ jobs:
pip install -r requirements-dev.txt
pip install .
- name: Type-checking with mypy
run: |
mypy .
run: mypy .
- name: Style-checking with black
run: black --check .
- name: Style-checking with flake8
run: flake8 .
4 changes: 2 additions & 2 deletions .github/workflows/unittests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: [3.5, 3.6, 3.7]
python-version: [3.6, 3.7, 3.8]

steps:
- uses: actions/checkout@v2
Expand All @@ -30,4 +30,4 @@ jobs:
pip install .
- name: Test with pytest
run: |
pytest
pytest
7 changes: 7 additions & 0 deletions .isort.cfg
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
[settings]
known_third_party = PyPDF2,backports,camelot,celery,click,cv2,flask,pandas,pygments,setuptools,six,sqlalchemy,werkzeug
line_length=79
indent=' '
multi_line_output=3
length_sort=0
include_trailing_comma=True
35 changes: 35 additions & 0 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
# pre-commit run --all-files
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v3.2.0
hooks:
- id: end-of-file-fixer
- id: mixed-line-ending
- repo: https://github.com/pre-commit/mirrors-mypy
rev: v0.782
hooks:
- id: mypy
exclude: docs/
- repo: https://github.com/asottile/seed-isort-config
rev: v2.2.0
hooks:
- id: seed-isort-config
- repo: https://github.com/pre-commit/mirrors-isort
rev: v5.5.1
hooks:
- id: isort
- repo: https://github.com/asottile/pyupgrade
rev: v2.7.2
hooks:
- id: pyupgrade
args: [--py36-plus]
- repo: https://github.com/psf/black
rev: 20.8b1
hooks:
- id: black
exclude: docs/
- repo: https://github.com/asottile/blacken-docs
rev: v1.8.0
hooks:
- id: blacken-docs
additional_dependencies: [black==20.8b1]
3 changes: 0 additions & 3 deletions arthur.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,7 @@
# -*- coding: utf-8 -*-

import multiprocessing

from excalibur.cli import webserver


if __name__ == "__main__":
multiprocessing.freeze_support()
webserver()
2 changes: 1 addition & 1 deletion docs/_templates/hacks.html
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,4 @@
/* Have code blocks escape the document right-margin. */
div.highlight pre {margin-right: -30px;}
}
</style>
</style>
2 changes: 1 addition & 1 deletion docs/_templates/sidebarlogo.html
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,4 @@
<p>
<iframe src="https://ghbtns.com/github-btn.html?user=camelot-dev&repo=excalibur&type=watch&count=true&size=large"
allowtransparency="true" frameborder="0" scrolling="0" width="200px" height="35px"></iframe>
</p>
</p>
2 changes: 1 addition & 1 deletion docs/_themes/.gitignore
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
*.pyc
*.pyo
*.pyo
2 changes: 1 addition & 1 deletion docs/_themes/LICENSE
Original file line number Diff line number Diff line change
Expand Up @@ -34,4 +34,4 @@ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS THEME, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
POSSIBILITY OF SUCH DAMAGE.
16 changes: 14 additions & 2 deletions docs/_themes/flask_theme_support.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,19 @@
# flasky pygments style based on tango style
from pygments.style import Style
from pygments.token import Keyword, Name, Comment, String, Error, \
Number, Operator, Generic, Whitespace, Punctuation, Other, Literal
from pygments.token import (
Comment,
Error,
Generic,
Keyword,
Literal,
Name,
Number,
Operator,
Other,
Punctuation,
String,
Whitespace,
)


class FlaskyStyle(Style):
Expand Down
3 changes: 1 addition & 2 deletions docs/conf.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
#
# Configuration file for the Sphinx documentation builder.
#
Expand All @@ -14,12 +13,12 @@

import os
import sys

sys.path.insert(0, os.path.abspath('..'))
sys.path.insert(0, os.path.abspath('_themes'))

import excalibur


# -- Project information -----------------------------------------------------

project = 'Excalibur'
Expand Down
4 changes: 1 addition & 3 deletions excalibur/__init__.py
Original file line number Diff line number Diff line change
@@ -1,3 +1 @@
# -*- coding: utf-8 -*-

from .__version__ import __version__
from .__version__ import __version__ # noqa
5 changes: 0 additions & 5 deletions excalibur/__main__.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,3 @@
# -*- coding: utf-8 -*-

from __future__ import absolute_import


__all__ = ("main",)


Expand Down
6 changes: 2 additions & 4 deletions excalibur/__version__.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
# -*- coding: utf-8 -*-

VERSION = (0, 4, 3)
PRERELEASE = None # alpha, beta or rc
REVISION = None
Expand All @@ -8,9 +6,9 @@
def generate_version(version, prerelease=None, revision=None):
version_parts = [".".join(map(str, version))]
if prerelease is not None:
version_parts.append("-{}".format(prerelease))
version_parts.append(f"-{prerelease}")
if revision is not None:
version_parts.append(".{}".format(revision))
version_parts.append(f".{revision}")
return "".join(version_parts)


Expand Down
8 changes: 4 additions & 4 deletions excalibur/cli.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,13 @@
# -*- coding: utf-8 -*-

import os
from threading import Timer

import click

from . import __version__, settings
from . import __version__
from . import configuration as conf
from . import settings
from .operators.python_operator import PythonOperator
from .tasks import split, extract
from .tasks import extract, split
from .utils.database import initialize_database, reset_database
from .www.app import create_app

Expand Down Expand Up @@ -68,6 +67,7 @@ def open_browser():
@cli.command("worker")
def worker(*args, **kwargs):
from celery.bin import worker

from .executors.celery_executor import app as celery_app

worker = worker.worker(app=celery_app)
Expand Down
1 change: 0 additions & 1 deletion excalibur/config_templates/__init__.py
Original file line number Diff line number Diff line change
@@ -1 +0,0 @@
# -*- coding: utf-8 -*-
1 change: 0 additions & 1 deletion excalibur/config_templates/default_celery.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
from .. import configuration as conf


broker_url = conf.get("celery", "BROKER_URL")

DEFAULT_CELERY_CONFIG = {
Expand Down
25 changes: 8 additions & 17 deletions excalibur/configuration.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
# -*- coding: utf-8 -*-

import os
import six

import six
from backports.configparser import ConfigParser


Expand All @@ -13,13 +11,8 @@ def _read_default_config_file(file_name):
"""
templates_dir = os.path.join(os.path.dirname(__file__), "config_templates")
file_path = os.path.join(templates_dir, file_name)
if six.PY2:
with open(file_path) as f:
config = f.read()
return config.decode("utf-8")
else:
with open(file_path, encoding="utf-8") as f:
return f.read()
with open(file_path, encoding="utf-8") as f:
return f.read()


def expand_env_var(env_var):
Expand All @@ -46,7 +39,7 @@ def expand_env_var(env_var):

class ExcaliburConfigParser(ConfigParser):
def __init__(self, default_config=None, *args, **kwargs):
super(ExcaliburConfigParser, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)

self.excalibur_defaults = ConfigParser(*args, **kwargs)
if default_config is not None:
Expand All @@ -68,10 +61,8 @@ def get(self, section, key, **kwargs):
section = str(section).lower()
key = str(key).lower()

if super(ExcaliburConfigParser, self).has_option(section, key):
return expand_env_var(
super(ExcaliburConfigParser, self).get(section, key, **kwargs)
)
if super().has_option(section, key):
return expand_env_var(super().get(section, key, **kwargs))

if self.excalibur_defaults.has_option(section, key):
return expand_env_var(self.excalibur_defaults.get(section, key, **kwargs))
Expand All @@ -83,7 +74,7 @@ def get(self, section, key, **kwargs):
)

def read(self, filename):
super(ExcaliburConfigParser, self).read(filename)
super().read(filename)
self._validate()


Expand Down Expand Up @@ -119,7 +110,7 @@ def parameterized_config(template):


if not os.path.isfile(EXCALIBUR_CONFIG):
print("Creating new Excalibur configuration file in: {}".format(EXCALIBUR_CONFIG))
print(f"Creating new Excalibur configuration file in: {EXCALIBUR_CONFIG}")
with open(EXCALIBUR_CONFIG, "w") as f:
cfg = parameterized_config(DEFAULT_CONFIG)
if six.PY2:
Expand Down
5 changes: 1 addition & 4 deletions excalibur/executors/__init__.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,8 @@
# -*- coding: utf-8 -*-

import atexit

from .. import configuration as conf
from .celery_executor import CeleryExecutor
from .sequential_executor import SequentialExecutor
from .. import configuration as conf


DEFAULT_EXECUTOR = None

Expand Down
5 changes: 1 addition & 4 deletions excalibur/executors/base_executor.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,4 @@
# -*- coding: utf-8 -*-


class BaseExecutor(object):
class BaseExecutor:
def __init__(self):
pass

Expand Down
9 changes: 3 additions & 6 deletions excalibur/executors/celery_executor.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,12 @@
# -*- coding: utf-8 -*-

import traceback
import subprocess
import traceback

from celery import Celery

from .base_executor import BaseExecutor
from .. import configuration as conf
from ..config_templates.default_celery import DEFAULT_CELERY_CONFIG
from ..utils.module_loading import import_string

from .base_executor import BaseExecutor

if conf.has_option("celery", "celery_config_options"):
celery_configuration = import_string(conf.get("celery", "celery_config_options"))
Expand All @@ -27,7 +24,7 @@ def execute_command(command):
try:
subprocess.check_call(command, stderr=subprocess.STDOUT, close_fds=True)
except Exception as e:
traceback.print_exc()
traceback.print_exc(e)


class CeleryExecutor(BaseExecutor):
Expand Down
6 changes: 2 additions & 4 deletions excalibur/executors/sequential_executor.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
# -*- coding: utf-8 -*-

import traceback
import subprocess
import traceback
from concurrent.futures import ProcessPoolExecutor

from .base_executor import BaseExecutor
Expand All @@ -19,7 +17,7 @@ def execute_command(command):
task_id = command[-1]
_run(task_name, task_id)
except Exception as e:
traceback.print_exc()
traceback.print_exc(e)


class SequentialExecutor(BaseExecutor):
Expand Down
15 changes: 10 additions & 5 deletions excalibur/models.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,18 @@
# -*- coding: utf-8 -*-

import json
import datetime as dt
import json
from typing import Any # noqa

from sqlalchemy import Boolean, Column, DateTime, Integer, String, Text, ForeignKey
from sqlalchemy import (
Boolean,
Column,
DateTime,
ForeignKey,
Integer,
String,
Text,
)
from sqlalchemy.ext.declarative import declarative_base


Base = declarative_base() # type: Any
ID_LEN = 100
STR_LEN = 500
Expand Down
1 change: 0 additions & 1 deletion excalibur/operators/__init__.py
Original file line number Diff line number Diff line change
@@ -1 +0,0 @@
# -*- coding: utf-8 -*-
5 changes: 1 addition & 4 deletions excalibur/operators/base_operator.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,4 @@
# -*- coding: utf-8 -*-


class BaseOperator(object):
class BaseOperator:
def __init__(self):
pass

Expand Down
2 changes: 0 additions & 2 deletions excalibur/operators/python_operator.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
# -*- coding: utf-8 -*-

from .base_operator import BaseOperator


Expand Down
Loading

0 comments on commit 425eaaf

Please sign in to comment.