feat(sourcecode): add setuptools build metadata (DataDog#5292)

Adds automatic instrumentation for setuptools for embedding git metadata of the Python package.

This is implemented by monkey-patching setuptools.setup to add source_code_url as an additional entry in the project_urls metadata field.


Co-authored-by: Tahir H. Butt <[email protected]>
Co-authored-by: Ofek Lev <[email protected]>
Co-authored-by: Kyle Verhoog <[email protected]>

.circleci/config.yml

@@ -393,6 +393,12 @@ jobs:
           # We skip the test_http.py tests in riot and run the test_http.py tests through tox.
           pattern: '^py.\+-tracer_test_http'
 
+  sourcecode:
+    <<: *contrib_job
+    steps:
+      - run_test:
+          pattern: "sourcecode"
+
   telemetry:
     <<: *machine_executor
     steps:
@@ -1208,6 +1214,7 @@ requires_tests: &requires_tests
     - sanic
     - snowflake
     - sqlalchemy
+    - sourcecode 
     - starlette
     - stdlib
     - test_logging
@@ -1313,6 +1320,7 @@ workflows:
       - starlette: *requires_base_venvs
       - stdlib: *requires_base_venvs
       - sqlalchemy: *requires_base_venvs
+      - sourcecode: *requires_base_venvs
       - test_logging: *requires_base_venvs
       - tornado: *requires_base_venvs
       - tracer: *requires_base_venvs

ddtrace/sourcecode/__init__.py

@@ -0,0 +1,36 @@
+"""
+`Datadog Source Code Integration`__ is supported for Git by the addition of the
+repository URL and commit hash in the Python package metadata field
+``Project-URL`` with name ``source_code_link``.
+
+Format of ``source_code_link``: ``<repository url>#<commit hash>``
+
+
+setuptools
+----------
+
+The ``ddtrace`` provides automatic instrumentation of ``setuptools`` to embed
+the source code link into the project metadata. ``ddtrace`` has to be installed
+as a build dependency.
+
+Packages with ``pyproject.toml`` can update the build system requirements::
+
+  [build-system]
+  requires = ["setuptools", "ddtrace"]
+  build-backend = "setuptools.build_meta"
+
+
+The instrumentation of ``setuptools`` can be automatically enabled to embed the
+source code link with a one-line import in ``setup.py`` (before setuptools import)::
+
+   import ddtrace.sourcecode.setuptools_auto
+   from setuptools import setup
+
+   setup(
+       name="mypackage",
+       version="0.0.1",
+       #...
+   )
+
+.. __: https://docs.datadoghq.com/integrations/guide/source-code-integration/
+"""

ddtrace/sourcecode/_utils.py

@@ -0,0 +1,73 @@
+import re
+import subprocess
+
+from ddtrace.internal.compat import parse
+
+
+SCP_REGEXP = re.compile("^[a-z0-9_]+@([a-z0-9._-]+):(.*)$", re.IGNORECASE)
+
+
+def _remove_suffix(s, suffix):
+    if s.endswith(suffix):
+        return s[: -len(suffix)]
+    else:
+        return s
+
+
+def normalize_repository_url(url):
+    scheme = ""
+    hostname = ""
+    port = None
+    path = ""
+
+    match = SCP_REGEXP.match(url)
+    if match:
+        # Check URLs like "[email protected]:user/project.git",
+        scheme = "https"
+        hostname = match.group(1)
+        path = "/" + match.group(2)
+    else:
+        u = parse.urlsplit(url)
+        if u.scheme == "" and u.hostname is None:
+            # Try to add a scheme.
+            u = parse.urlsplit("https://" + url)  # Default to HTTPS.
+            if u.hostname is None:
+                return ""
+
+        scheme = u.scheme
+        hostname = u.hostname
+        port = u.port
+        path = u.path
+
+        if scheme not in ("http", "https", "git", "ssh"):
+            return ""
+
+        if not scheme.startswith("http"):
+            scheme = "https"  # Default to HTTPS.
+            port = None
+
+    path = _remove_suffix(path, ".git/")
+    path = _remove_suffix(path, ".git")
+
+    netloc = hostname
+    if port is not None:
+        netloc += ":" + str(port)
+
+    return parse.urlunsplit((scheme, netloc, path, "", ""))
+
+
+def _query_git(args):
+    ver = subprocess.Popen(["git"] + args, stdout=subprocess.PIPE).communicate()[0]
+    return ver.strip().decode("utf-8")
+
+
+def get_commit_sha():
+    return _query_git(["rev-parse", "HEAD"])
+
+
+def get_repository_url():
+    return _query_git(["config", "--get", "remote.origin.url"])
+
+
+def get_source_code_link():
+    return normalize_repository_url(get_repository_url()) + "#" + get_commit_sha()

ddtrace/sourcecode/setuptools_auto.py

@@ -0,0 +1,34 @@
+from ddtrace.vendor.wrapt import wrap_function_wrapper as _w
+
+
+try:
+    import distutils.core as distutils_core
+
+    import setuptools
+except ImportError:
+    distutils_core = None  # type: ignore[assignment]
+    setuptools = None  # type: ignore[assignment]
+
+
+from ._utils import get_source_code_link
+
+
+def _setup(wrapped, instance, args, kwargs):
+    source_code_link = get_source_code_link()
+
+    if "project_urls" not in kwargs:
+        kwargs["project_urls"] = {}
+
+    kwargs["project_urls"]["source_code_link"] = source_code_link
+
+    return wrapped(*args, **kwargs)
+
+
+def _patch():
+
+    if distutils_core and setuptools:
+        _w(distutils_core, "setup", _setup)
+        _w(setuptools, "setup", _setup)
+
+
+_patch()

docs/api.rst

@@ -46,3 +46,10 @@ Dynamic Instrumentation
 
 .. automodule:: ddtrace.debugging
     :members:
+
+
+Source Code Integration
+=======================
+
+.. automodule:: ddtrace.sourcecode
+    :members:

releasenotes/notes/sourcecode-setuptools-b869c154bbb897d9.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - |
+    Adds source code integration with setuptools build metadata. This enables traces and profiles to be automatically tagged with git metadata to track deployments in Datadog.

riotfile.py

@@ -2596,5 +2596,13 @@ def select_pys(min_version=MIN_PYTHON_VERSION, max_version=MAX_PYTHON_VERSION):
                 "pytest-asyncio": latest,
             },
         ),
+        Venv(
+            name="sourcecode",
+            command="pytest {cmdargs} tests/sourcecode",
+            pys=select_pys(),
+            pkgs={
+                "setuptools": ["<=67.6.0"],
+            },
+        ),
     ],
 )

tests/sourcecode/conftest.py

@@ -0,0 +1,85 @@
+import contextlib
+import os
+import shutil
+import subprocess
+import tempfile
+
+import pytest
+
+
+def _run(cmd):
+    return subprocess.check_output(cmd, shell=True)
+
+
+@contextlib.contextmanager
+def create_package(directory, pyproject, setup):
+    package_dir = os.path.join(directory, "mypackage")
+    os.mkdir(package_dir)
+
+    pyproject_file = os.path.join(package_dir, "pyproject.toml")
+    with open(pyproject_file, "wb") as f:
+        f.write(pyproject.encode("utf-8"))
+
+    setup_file = os.path.join(package_dir, "setup.py")
+    with open(setup_file, "wb") as f:
+        f.write(setup.encode("utf-8"))
+
+    _ = os.path.join(package_dir, "mypackage")
+    os.mkdir(_)
+    with open(os.path.join(_, "__init__.py"), "wb") as f:
+        f.write('"0.0.1"'.encode("utf-8"))
+
+    cwd = os.getcwd()
+    os.chdir(package_dir)
+
+    try:
+        _run("git init")
+        _run("git config --local user.name user")
+        _run("git config --local user.email [email protected]")
+        _run("git add .")
+        _run("git commit -m init")
+        _run("git remote add origin https://github.com/companydotcom/repo.git")
+
+        yield package_dir
+    finally:
+        os.chdir(cwd)
+
+        try:
+            shutil.rmtree(package_dir)
+        except OSError:
+            pass
+
+
+@pytest.fixture
+def temp_dir():
+    d = tempfile.mkdtemp()
+    try:
+        d = os.path.realpath(d)
+        yield d
+    finally:
+        try:
+            shutil.rmtree(d)
+        except OSError:
+            pass
+
+
+@pytest.fixture
+def example(temp_dir):
+    with create_package(
+        temp_dir,
+        """\
+[build-system]
+requires = ["setuptools", "ddtrace"]
+build-backend = "setuptools.build_meta"
+""",
+        """\
+import ddtrace.sourcecode.setuptools_auto
+from setuptools import setup
+
+setup(
+    name="mypackage",
+    version="0.0.1",
+)
+""",
+    ) as package:
+        yield package

tests/sourcecode/test_repository_url.py

@@ -0,0 +1,79 @@
+import pytest
+
+from ddtrace.sourcecode._utils import normalize_repository_url
+
+
+@pytest.mark.parametrize(
+    "repository_url,expected",
+    [
+        # Supported schemes.
+        (
+            "http://github.com/user/project.git",
+            "http://github.com/user/project",
+        ),
+        (
+            "https://github.com/user/project.git",
+            "https://github.com/user/project",
+        ),
+        (
+            "git://github.com/user/project.git",
+            "https://github.com/user/project",
+        ),
+        (
+            "[email protected]:user/project.git",
+            "https://github.com/user/project",
+        ),
+        (
+            "ssh://[email protected]/user/project.git",
+            "https://github.com/user/project",
+        ),
+        (
+            "git://github.com/user/project.git/",
+            "https://github.com/user/project",
+        ),
+        # No scheme but valid TLD.
+        (
+            "github.com/user/project",
+            "https://github.com/user/project",
+        ),
+        # Subdomain preserved.
+        (
+            "http://www.github.com/user/project.git",
+            "http://www.github.com/user/project",
+        ),
+        # Preserve port for HTTP/HTTPS schemes.
+        (
+            "http://github.com:8080/user/project.git",
+            "http://github.com:8080/user/project",
+        ),
+        (
+            "https://github.com:8080/user/project.git",
+            "https://github.com:8080/user/project",
+        ),
+        # Do not preserve port otherwise.
+        (
+            "ssh://[email protected]:22/user/project.git",
+            "https://github.com/user/project",
+        ),
+        # Strip credentials.
+        (
+            "https://gitlab-ci-token:[email protected]/user/project.git",
+            "https://gitlab.com/user/project",
+        ),
+        # Not supported.
+        (
+            "ftp:///path/to/repo.git/",
+            "",
+        ),
+        (
+            "/path/to/repo.git/",
+            "",
+        ),
+        (
+            "file:///path/to/repo.git/",
+            "",
+        ),
+    ],
+)
+def test_normalize_repository_url(repository_url, expected):
+    assert normalize_repository_url(repository_url) == expected

tests/sourcecode/test_setuptools.py

@@ -0,0 +1,18 @@
+import os
+import subprocess
+
+
+def test_example(example):
+    git_sha = subprocess.check_output("git rev-parse HEAD", shell=True).decode("utf-8").strip()
+    expected = "Project-URL: source_code_link, https://github.com/companydotcom/repo#{}".format(git_sha)
+    subprocess.check_output("python setup.py bdist", shell=True)
+    pkg_info = os.path.join(
+        example,
+        "mypackage.egg-info",
+        "PKG-INFO",
+    )
+    with open(pkg_info, "r") as f:
+        links = [_.strip() for _ in f.readlines() if _.startswith("Project-URL: source_code_link, ")]
+
+    assert len(links) == 1
+    assert links[0] == expected