Skip to content

Commit

Permalink
Use f-strings and fix py.test coverage scope
Browse files Browse the repository at this point in the history
  • Loading branch information
joowani committed Feb 17, 2021
1 parent 9196b8b commit 46fd36e
Show file tree
Hide file tree
Showing 8 changed files with 58 additions and 43 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/build.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ jobs:
- name: Run mypy
run: mypy kq
- name: Run pytest
run: py.test --cov=./ --cov-report=xml
run: py.test --cov=kq --cov-report=xml
- name: Run Sphinx doctest
run: python -m sphinx -b doctest docs docs/_build
- name: Run Sphinx HTML
Expand Down
2 changes: 1 addition & 1 deletion .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ repos:
rev: v0.790
hooks:
- id: mypy
args: [ kq ]
files: ^kq/
- repo: https://gitlab.com/pycqa/flake8
rev: 3.8.4
hooks:
Expand Down
4 changes: 2 additions & 2 deletions kq/queue.py
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ def enqueue(
key=key,
partition=part,
)
self._logger.info("Enqueueing {} ...".format(job))
self._logger.info(f"Enqueueing {job} ...")
self._producer.send(
self._topic,
value=self._serializer(job),
Expand Down Expand Up @@ -210,7 +210,7 @@ def __repr__(self) -> str:
:return: String representation of the queue.
:rtype: str
"""
return "Queue(hosts={}, topic={})".format(self._hosts, self._topic)
return f"Queue(hosts={self._hosts}, topic={self._topic})"

def __del__(self) -> None: # pragma: no covers
# noinspection PyBroadException
Expand Down
7 changes: 4 additions & 3 deletions kq/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
from typing import Any, Callable


# noinspection PyUnresolvedReferences
def get_call_repr(func: Callable[..., Any], *args: Any, **kwargs: Any) -> str:
"""Return the string representation of the function call.
Expand All @@ -15,16 +16,16 @@ def get_call_repr(func: Callable[..., Any], *args: Any, **kwargs: Any) -> str:
"""
# Functions, builtins and methods
if ismethod(func) or isfunction(func) or isbuiltin(func):
func_repr = "{}.{}".format(func.__module__, func.__qualname__)
func_repr = f"{func.__module__}.{func.__qualname__}"
# A callable class instance
elif not isclass(func) and hasattr(func, "__call__"):
func_repr = "{}.{}".format(func.__module__, func.__class__.__name__)
func_repr = f"{func.__module__}.{func.__class__.__name__}"
else:
func_repr = repr(func)

args_reprs = [repr(arg) for arg in args]
kwargs_reprs = [k + "=" + repr(v) for k, v in sorted(kwargs.items())]
return "{}({})".format(func_repr, ", ".join(args_reprs + kwargs_reprs))
return f'{func_repr}({", ".join(args_reprs + kwargs_reprs)})'


def is_none_or_logger(obj: Any) -> bool:
Expand Down
20 changes: 9 additions & 11 deletions kq/worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,9 +87,7 @@ def __repr__(self) -> str:
:return: String representation of the worker.
:rtype: str
"""
return "Worker(hosts={}, topic={}, group={})".format(
self._hosts, self._topic, self._group
)
return f"Worker(hosts={self._hosts}, topic={self._topic}, group={self._group})"

def __del__(self) -> None: # pragma: no cover
# noinspection PyBroadException
Expand Down Expand Up @@ -129,8 +127,8 @@ def _execute_callback(
try:
self._logger.info("Executing callback ...")
self._callback(status, message, job, res, err, stacktrace)
except Exception as e:
self._logger.exception("Callback raised an exception: {}".format(e))
except Exception as err:
self._logger.exception(f"Callback raised an exception: {err}")

def _process_message(self, msg: Message) -> None:
"""De-serialize the message and execute the job.
Expand All @@ -148,10 +146,10 @@ def _process_message(self, msg: Message) -> None:
job_repr = get_call_repr(job.func, *job.args, **job.kwargs)

except Exception as err:
self._logger.exception("Job was invalid: {}".format(err))
self._logger.exception(f"Job was invalid: {err}")
self._execute_callback("invalid", msg, None, None, None, None)
else:
self._logger.info("Executing job {}: {}".format(job.id, job_repr))
self._logger.info(f"Executing job {job.id}: {job_repr}")

timer: Optional[threading.Timer]
if job.timeout:
Expand All @@ -162,14 +160,14 @@ def _process_message(self, msg: Message) -> None:
try:
res = job.func(*job.args, **job.kwargs)
except KeyboardInterrupt:
self._logger.error("Job {} timed out or was interrupted".format(job.id))
self._logger.error(f"Job {job.id} timed out or was interrupted")
self._execute_callback("timeout", msg, job, None, None, None)
except Exception as err:
self._logger.exception("Job {} raised an exception:".format(job.id))
self._logger.exception(f"Job {job.id} raised an exception:")
tb = traceback.format_exc()
self._execute_callback("failure", msg, job, None, err, tb)
else:
self._logger.info("Job {} returned: {}".format(job.id, res))
self._logger.info(f"Job {job.id} returned: {res}")
self._execute_callback("success", msg, job, res, None, None)
finally:
if timer is not None:
Expand Down Expand Up @@ -243,7 +241,7 @@ def start(
:return: Total number of messages processed.
:rtype: int
"""
self._logger.info("Starting {} ...".format(self))
self._logger.info(f"Starting {self} ...")

self._consumer.unsubscribe()
self._consumer.subscribe([self.topic])
Expand Down
2 changes: 1 addition & 1 deletion tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ def __call__(self, status, message, job, result, exception, stacktrace):
assert exception is not None
assert stacktrace is not None

self.logger.info('Callback got job status "{}"'.format(status))
self.logger.info(f'Callback got job status "{status}"')


class Deserializer(object):
Expand Down
28 changes: 22 additions & 6 deletions tests/test_queue.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ def test_queue_enqueue_function(queue, func, topic, log):
assert job.timeout == 0
assert job.key is None
assert job.partition is None
assert log.last_line == "[INFO] Enqueueing {} ...".format(job)
assert log.last_line == f"[INFO] Enqueueing {job} ..."


def test_queue_enqueue_function_with_spec(func, queue, topic, log):
Expand All @@ -72,10 +72,9 @@ def test_queue_enqueue_function_with_spec(func, queue, topic, log):
assert job.timeout == 0
assert job.key == b"foo"
assert job.partition == 0
assert log.last_line == "[INFO] Enqueueing {} ...".format(job)
assert log.last_line == f"[INFO] Enqueueing {job} ..."


# noinspection PyTypeChecker
def test_queue_enqueue_function_with_bad_args(func, queue):
with pytest.raises(AssertionError) as e:
queue.enqueue(1)
Expand Down Expand Up @@ -120,7 +119,7 @@ def test_queue_enqueue_job_fully_populated(func, queue, topic, log):
assert job.timeout == 10
assert job.key == b"bar"
assert job.partition == 0
assert log.last_line.startswith("[INFO] Enqueueing {} ...".format(job))
assert log.last_line.startswith(f"[INFO] Enqueueing {job} ...")


def test_queue_enqueue_job_partially_populated(func, queue, topic, log):
Expand All @@ -137,7 +136,24 @@ def test_queue_enqueue_job_partially_populated(func, queue, topic, log):
assert job.timeout == 0
assert job.key is None
assert job.partition is None
assert log.last_line.startswith("[INFO] Enqueueing {} ...".format(job))
assert log.last_line.startswith(f"[INFO] Enqueueing {job} ...")


def test_queue_enqueue_job_with_no_args(func, queue, topic, log):
job = Job(func=func)

job = queue.enqueue(job)
assert isinstance(job, Job)
assert isinstance(job.id, str)
assert isinstance(job.timestamp, int)
assert job.topic == topic
assert job.func == func
assert job.args == tuple()
assert job.kwargs == {}
assert job.timeout == 0
assert job.key is None
assert job.partition is None
assert log.last_line.startswith(f"[INFO] Enqueueing {job} ...")


def test_queue_enqueue_job_with_spec(func, queue, topic, log):
Expand Down Expand Up @@ -168,7 +184,7 @@ def test_queue_enqueue_job_with_spec(func, queue, topic, log):
assert job.timeout == 10
assert job.key == b"bar"
assert job.partition == 0
assert log.last_line.startswith("[INFO] Enqueueing {} ...".format(job))
assert log.last_line.startswith(f"[INFO] Enqueueing {job} ...")


def test_queue_enqueue_job_with_bad_args(func, queue, topic):
Expand Down
36 changes: 18 additions & 18 deletions tests/test_worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,11 +53,11 @@ def test_worker_run_success_function(queue, worker, success_func, log):
worker.start(max_messages=1)

out = log.last_lines(7)
assert next(out).startswith("[INFO] Enqueueing {}".format(job))
assert next(out).startswith("[INFO] Starting {}".format(worker))
assert next(out).startswith(f"[INFO] Enqueueing {job}")
assert next(out).startswith(f"[INFO] Starting {worker}")
assert next(out).startswith("[INFO] Processing Message")
assert next(out).startswith("[INFO] Executing job {}".format(job.id))
assert next(out).startswith("[INFO] Job {} returned: 2".format(job.id))
assert next(out).startswith(f"[INFO] Executing job {job.id}")
assert next(out).startswith(f"[INFO] Job {job.id} returned: 2")
assert next(out).startswith("[INFO] Executing callback")
assert next(out).startswith('[INFO] Callback got job status "success"')

Expand All @@ -67,11 +67,11 @@ def test_worker_run_failure_function(queue, worker, failure_func, log):
worker.start(max_messages=1)

out = log.last_lines(7)
assert next(out).startswith("[INFO] Enqueueing {}".format(job))
assert next(out).startswith("[INFO] Starting {}".format(worker))
assert next(out).startswith(f"[INFO] Enqueueing {job}")
assert next(out).startswith(f"[INFO] Starting {worker}")
assert next(out).startswith("[INFO] Processing Message")
assert next(out).startswith("[INFO] Executing job {}".format(job.id))
assert next(out).startswith("[ERROR] Job {} raised".format(job.id))
assert next(out).startswith(f"[INFO] Executing job {job.id}")
assert next(out).startswith(f"[ERROR] Job {job.id} raised")
assert next(out).startswith("[INFO] Executing callback")
assert next(out).startswith('[INFO] Callback got job status "failure"')

Expand All @@ -81,11 +81,11 @@ def test_worker_run_timeout_function(queue, worker, timeout_func, log):
worker.start(max_messages=1)

out = log.last_lines(7)
assert next(out).startswith("[INFO] Enqueueing {}".format(job))
assert next(out).startswith("[INFO] Starting {}".format(worker))
assert next(out).startswith(f"[INFO] Enqueueing {job}")
assert next(out).startswith(f"[INFO] Starting {worker}")
assert next(out).startswith("[INFO] Processing Message")
assert next(out).startswith("[INFO] Executing job {}".format(job.id))
assert next(out).startswith("[ERROR] Job {} timed out".format(job.id))
assert next(out).startswith(f"[INFO] Executing job {job.id}")
assert next(out).startswith(f"[ERROR] Job {job.id} timed out")
assert next(out).startswith("[INFO] Executing callback")
assert next(out).startswith('[INFO] Callback got job status "timeout"')

Expand All @@ -96,11 +96,11 @@ def test_worker_run_bad_callback(queue, worker, success_func, callback, log):
worker.start(max_messages=1)

out = log.last_lines(7)
assert next(out).startswith("[INFO] Enqueueing {}".format(job))
assert next(out).startswith("[INFO] Starting {}".format(worker))
assert next(out).startswith(f"[INFO] Enqueueing {job}")
assert next(out).startswith(f"[INFO] Starting {worker}")
assert next(out).startswith("[INFO] Processing Message")
assert next(out).startswith("[INFO] Executing job {}".format(job.id))
assert next(out).startswith("[INFO] Job {} returned: 20".format(job.id))
assert next(out).startswith(f"[INFO] Executing job {job.id}")
assert next(out).startswith(f"[INFO] Job {job.id} returned: 20")
assert next(out).startswith("[INFO] Executing callback")
assert next(out).startswith("[ERROR] Callback raised an exception")

Expand All @@ -111,8 +111,8 @@ def test_worker_run_bad_job(queue, worker, success_func, deserializer, log):
worker.start(max_messages=1)

out = log.last_lines(6)
assert next(out).startswith("[INFO] Enqueueing {}".format(job))
assert next(out).startswith("[INFO] Starting {}".format(worker))
assert next(out).startswith(f"[INFO] Enqueueing {job}")
assert next(out).startswith(f"[INFO] Starting {worker}")
assert next(out).startswith("[INFO] Processing Message")
assert next(out).startswith("[ERROR] Job was invalid")
assert next(out).startswith("[INFO] Executing callback")
Expand Down

0 comments on commit 46fd36e

Please sign in to comment.