Skip to content

Commit

Permalink
Tweak Coveralls settings (sqlfluff#4199)
Browse files Browse the repository at this point in the history
* Add .gitpod.yml

* Remove .gitpod.yml

* Tweak Coveralls

* Tweak base-path

* Trigger build

* Specify "--cov=" appropriate to each build

* Tweaks

* Tweaks

* Change .tox paths to src/sqlfluff paths

* Run patch_lcov.py from tox, not from ci-tests.yml

* Fail-safe the script

* Add debug message

* Do nothing if coverage file not found

* Only try and patch lines where path begins with ".tox"

* Move Coveralls coverage builds first so we get results sooner

* Revert "Move Coveralls coverage builds first so we get results sooner"

This reverts commit b5df02b.

* PR review: more detailed docstring
  • Loading branch information
barrywhart authored Dec 23, 2022
1 parent df09b33 commit e62e278
Show file tree
Hide file tree
Showing 7 changed files with 64 additions and 23 deletions.
13 changes: 3 additions & 10 deletions .github/workflows/ci-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ jobs:
pip install tox
- name: Run the tests
run: |
tox -e py -- -n 2 test
tox -e py -- --cov=sqlfluff -n 2 test
- name: Upload Coverage Report
uses: codecov/codecov-action@v3
with:
Expand Down Expand Up @@ -201,7 +201,7 @@ jobs:
pip install tox
- name: Run the tests
run: |
tox -e ${{ matrix.dbt-version }} -- plugins/sqlfluff-templater-dbt
tox -e ${{ matrix.dbt-version }} -- --cov=sqlfluff_templater_dbt plugins/sqlfluff-templater-dbt
- name: Upload Coverage Report
uses: codecov/codecov-action@v3
with:
Expand Down Expand Up @@ -243,20 +243,13 @@ jobs:
# working dir on D drive which causes problems.
run: |
mkdir temp_pytest
python -m tox -e winpy -- -n 2 test
python -m tox -e winpy -- --cov=sqlfluff -n 2 test
- name: Upload Coverage Report
uses: codecov/codecov-action@v3
with:
# Public token to allow coverage from forks to work.
token: fd44b020-c716-4f72-b5f3-03fa20119956
files: ./coverage.xml
- name: Coveralls Parallel
uses: coverallsapp/github-action@master
with:
path-to-lcov: coverage.lcov
github-token: ${{ secrets.github_token }}
flag-name: run-windows-${{ matrix.python-version }}
parallel: true
python-windows-dbt-tests:
runs-on: windows-latest
name: DBT Plugin Python 3.10 Windows tests
Expand Down
3 changes: 1 addition & 2 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
![SQLFluff](https://raw.githubusercontent.com/sqlfluff/sqlfluff/main/images/sqlfluff-wide.png)

[![Coverage Status](https://coveralls.io/repos/github/sqlfluff/sqlfluff/badge.svg?branch=main)](https://coveralls.io/github/sqlfluff/sqlfluff?branch=main)

# The SQL Linter for Humans

[![PyPi Version](https://img.shields.io/pypi/v/sqlfluff.svg?style=flat-square&logo=PyPi)](https://pypi.org/project/sqlfluff/)
Expand All @@ -11,6 +9,7 @@
[![PyPi Downloads](https://img.shields.io/pypi/dm/sqlfluff?style=flat-square)](https://pypi.org/project/sqlfluff/)

[![codecov](https://img.shields.io/codecov/c/gh/sqlfluff/sqlfluff.svg?style=flat-square&logo=Codecov)](https://codecov.io/gh/sqlfluff/sqlfluff)
[![Coveralls](https://coveralls.io/repos/github/sqlfluff/sqlfluff/badge.svg?branch=main)](https://coveralls.io/github/sqlfluff/sqlfluff?branch=main)
[![GitHub Workflow Status](https://img.shields.io/github/workflow/status/sqlfluff/sqlfluff/CI%20Tests?logo=github&style=flat-square)](https://github.com/sqlfluff/sqlfluff/actions/workflows/ci-tests.yml?query=branch%3Amain)
[![ReadTheDocs](https://img.shields.io/readthedocs/sqlfluff?style=flat-square&logo=Read%20the%20Docs)](https://sqlfluff.readthedocs.io)
[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg?style=flat-square)](https://github.com/psf/black)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,10 @@
DBT_VERSION_STRING = DBT_VERSION.to_version_string()
DBT_VERSION_TUPLE = (int(DBT_VERSION.major), int(DBT_VERSION.minor))

if DBT_VERSION_TUPLE >= (1, 3):
COMPILED_SQL_ATTRIBUTE = "compiled_code"
RAW_SQL_ATTRIBUTE = "raw_code"
else:
COMPILED_SQL_ATTRIBUTE = "compiled_sql"
RAW_SQL_ATTRIBUTE = "raw_sql"
COMPILED_SQL_ATTRIBUTE = (
"compiled_code" if DBT_VERSION_TUPLE >= (1, 3) else "compiled_sql"
)
RAW_SQL_ATTRIBUTE = "raw_code" if DBT_VERSION_TUPLE >= (1, 3) else "raw_sql"


class DbtTemplater(JinjaTemplater):
Expand Down Expand Up @@ -222,7 +220,7 @@ def _find_skip_reason(self, project, expected_node_path) -> Optional[str]:
for node in nodes:
if node.original_file_path == expected_node_path:
return "disabled"
return None
return None # pragma: no cover

def _unsafe_process(
self, fname: Optional[str], in_str: str, config: FluffConfig = None
Expand Down Expand Up @@ -263,7 +261,7 @@ def _unsafe_process(
# If injected SQL is present, it contains a better picture
# of what will actually hit the database (e.g. with tests).
# However it's not always present.
compiled_sql = node.injected_sql
compiled_sql = node.injected_sql # pragma: no cover
else:
compiled_sql = getattr(node, COMPILED_SQL_ATTRIBUTE)

Expand Down
13 changes: 13 additions & 0 deletions plugins/sqlfluff-templater-dbt/test/conftest.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
"""pytest fixtures."""
import dbt.flags
import pytest


@pytest.fixture(scope="session", autouse=True)
def dbt_flags():
"""Set dbt flags for dbt templater tests."""
# Setting this to True disables some code in dbt-core that randomly runs
# some test code in core/dbt/parser/models.py, ModelParser. render_update().
# We've seen occasional runtime errors from that code:
# TypeError: cannot pickle '_thread.RLock' object
dbt.flags.USE_EXPERIMENTAL_PARSER = True
4 changes: 3 additions & 1 deletion src/sqlfluff/core/templaters/jinja.py
Original file line number Diff line number Diff line change
Expand Up @@ -503,7 +503,9 @@ def slice_file(
if make_template is None:
# make_template() was not provided. Use the base class
# implementation instead.
return super().slice_file(raw_str, templated_str, config, **kwargs)
return super().slice_file(
raw_str, templated_str, config, **kwargs
) # pragma: no cover

templater_logger.info("Slicing File Template")
templater_logger.debug(" Raw String: %r", raw_str)
Expand Down
33 changes: 33 additions & 0 deletions test/patch_lcov.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
"""Replaces .tox/ paths in the lcov file with paths relative to repo root.
Context: When the CI build runs tests, it uses tox, which installs SQLFluff
in a virtual environment. Thus, the coverage.lcov file generated by the tests
contains paths to the virtual environment. This script replaces those paths
with paths relative to the repo root. This allows the lcov file to be used by
Coveralls. Without this, Coveralls has valid coverage info, but it generates
URLs that point to source files that don't exist in the SQLFluff GitHub repo.
For example, we want to change this:
SF:.tox/py/lib/python3.10/site-packages/sqlfluff/__init__.py
to this:
SF:src/sqlfluff/__init__.py
"""
import re
from pathlib import Path

path = Path("coverage.lcov")
if path.exists():
lines = path.read_text().splitlines()
modified_lines = []
for line in lines:
if line.startswith("SF:.tox"):
m = re.search(r"^(SF:).*(sqlfluff/.*)", line)
if m:
modified_lines.append(f"{m.group(1)}src/{m.group(2)}")
else:
print(f"Could not patch line: {line}")
modified_lines.append(line)
else:
modified_lines.append(line)
path.write_text("\n".join(modified_lines))
7 changes: 5 additions & 2 deletions tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,8 @@ commands =
# Clean up from previous tests
python {toxinidir}/util.py clean-tests
# Run tests
pytest -vv -rsfE --cov=sqlfluff --cov-report=xml --cov-report=lcov {posargs: {toxinidir}/test} -m "not integration_test"
pytest -vv -rsfE --cov-report=xml --cov-report=lcov {posargs: {toxinidir}/test} -m "not integration_test"
python test/patch_lcov.py

[testenv:cov-init]
setenv =
Expand Down Expand Up @@ -142,7 +143,9 @@ testpaths = test

[coverage:run]
source = src/sqlfluff
omit = src/sqlfluff/__main__.py
omit =
src/sqlfluff/__main__.py
plugins/sqlfluff-templater-dbt/sqlfluff_templater_dbt/osmosis/*.py

[coverage:report]
exclude_lines =
Expand Down

0 comments on commit e62e278

Please sign in to comment.