From 4a416ad84ce74f109a1a5fea8b0303a91d7cc927 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Thu, 9 Mar 2023 09:08:40 -0500 Subject: [PATCH] Refine provider docs (#106) * arrange architecture docs to reside before adding a provider for more context Signed-off-by: Alex Goodman * add example provider implementation Signed-off-by: Alex Goodman * add development shell Signed-off-by: Alex Goodman * update documentation with more details about provider makeup and configuration Signed-off-by: Alex Goodman * add a developer shell Signed-off-by: Alex Goodman * add .env file support Signed-off-by: Alex Goodman * not formatting of new provider steps Signed-off-by: Alex Goodman * add note about poetry shell session Signed-off-by: Alex Goodman * typo example provider title Signed-off-by: Alex Goodman --------- Signed-off-by: Alex Goodman --- .github/scripts/dev-shell.sh | 124 +++++++ .github/scripts/update-dev-db.sh | 38 ++ .gitignore | 8 + DEVELOPING.md | 508 ++++++++++++++++++--------- Makefile | 27 ++ docs/grype-db-actions.drawio | 1 + docs/vunnel+grype-db-workflow.drawio | 1 + docs/vunnel-run-workflow.drawio | 1 + example/.gitignore | 1 + example/README.md | 47 +++ example/awesome/__init__.py | 91 +++++ example/awesome/parser.py | 73 ++++ example/run.py | 60 ++++ poetry.lock | 241 +++++-------- pyproject.toml | 9 +- src/vunnel/provider.py | 9 + 16 files changed, 924 insertions(+), 315 deletions(-) create mode 100755 .github/scripts/dev-shell.sh create mode 100755 .github/scripts/update-dev-db.sh create mode 100644 docs/grype-db-actions.drawio create mode 100644 docs/vunnel+grype-db-workflow.drawio create mode 100644 docs/vunnel-run-workflow.drawio create mode 100644 example/.gitignore create mode 100644 example/README.md create mode 100644 example/awesome/__init__.py create mode 100644 example/awesome/parser.py create mode 100644 example/run.py diff --git a/.github/scripts/dev-shell.sh b/.github/scripts/dev-shell.sh new file mode 100755 index 00000000..7b2212b3 --- /dev/null +++ b/.github/scripts/dev-shell.sh @@ -0,0 +1,124 @@ +set -euo pipefail + +DEV_VUNNEL_PROVIDERS=$@ +GRYPE_CONFIG=$(pwd)/.grype.yaml +GRYPE_DB_CONFIG=$(pwd)/.grype-db.yaml +DEV_POETRY_ENV_PATH=$(poetry env info --path) + +BOLD="\033[1m" +UNDERLINE="\033[4m" +RED="\033[31m" +MAGENTA="\033[35m" +RESET="\033[0m" + +function step() { + echo "${MAGENTA}• $*${RESET} ..." +} + +function title() { + echo "${BOLD}$*${RESET}" +} + +function error() { + echo "${RED}$*${RESET}" +} + +if [ -z "$*" ] +then + error "At least one provider must be specified" + echo "examples:" + echo " make dev provider=\"nvd\"" + echo " make dev providers=\"oracle wolfi\"" + + exit 1 +fi + +set +u +if [ -n "${DEV_VUNNEL_SHELL:-}" ]; then + error "Already in a vunnel development shell" + exit 0 +fi +set -u + +function finish { + error "Unable to setup development shell. Bailing..." +} +trap finish EXIT + + +title "Entering vunnel development shell..." + +if [ -f .env ]; then + step "Sourcing .env file" + set -o allexport + source .env + set +o allexport +fi + +step "Configuring with providers: $DEV_VUNNEL_PROVIDERS" + +step "Writing grype config: $GRYPE_CONFIG" +cat << EOF > "$GRYPE_CONFIG" +check-for-app-update: false +db: + auto-update: false + validate-age: false + cache-dir: $(pwd)/.cache/grype +EOF +export GRYPE_CONFIG + +step "Writing grype-db config: $GRYPE_DB_CONFIG" +cat << EOF > "$GRYPE_DB_CONFIG" +pull: + parallelism: 1 +provider: + root: ./data + vunnel: + executor: local + env: + GITHUB_TOKEN: \$GITHUB_TOKEN + NVD_API_KEY: \$NVD_API_KEY + configs: +EOF +for provider in $DEV_VUNNEL_PROVIDERS; do + echo " - name: $provider" >> "$GRYPE_DB_CONFIG" +done +export GRYPE_DB_CONFIG + +step "Activating poetry virtual env: $DEV_POETRY_ENV_PATH" +source "$DEV_POETRY_ENV_PATH/bin/activate" + +pids="" + +step "Installing editable version of vunnel" +pip install -e . > /dev/null & +pids="$pids $!" + +step "Building grype" +make build-grype & +pids="$pids $!" + +step "Building grype-db" +make build-grype-db & +pids="$pids $!" + +wait $pids + +export PATH=${DEV_VUNNEL_BIN_DIR}:$PATH +export DEV_VUNNEL_SHELL=true + +echo +echo "Note: development builds ${UNDERLINE}grype${RESET} and ${UNDERLINE}grype-db${RESET} are now available in your path." +echo "To update these builds run '${UNDERLINE}make build-grype${RESET}' and '${UNDERLINE}make build-grype-db${RESET}' respectively." +echo "To run your provider and update the grype database run '${UNDERLINE}make update-db${RESET}'." +echo "Type '${UNDERLINE}exit${RESET}' to exit the development shell." + +# we were able to setup everything, no need to detect failures from this point on... +trap - EXIT + +$SHELL + +unset DEV_VUNNEL_SHELL +unset DEV_VUNNEL_PROVIDERS + +title "Exiting vunnel development shell 👋" diff --git a/.github/scripts/update-dev-db.sh b/.github/scripts/update-dev-db.sh new file mode 100755 index 00000000..0704b0dc --- /dev/null +++ b/.github/scripts/update-dev-db.sh @@ -0,0 +1,38 @@ +set -euo pipefail + +BIN_DIR=./bin +GRYPE=${BIN_DIR}/grype +GRYPE_DB=${BIN_DIR}/grype-db + +BOLD="\033[1m" +RED="\033[31m" +MAGENTA="\033[35m" +RESET="\033[0m" + +function step() { + echo "${MAGENTA}• $*${RESET} ..." +} + +function title() { + echo "${BOLD}$*${RESET}" +} + +function error() { + echo "${RED}$*${RESET}" +} + +step "Updating vunnel providers" +${GRYPE_DB} pull -v + +rm -rf build + +step "Building grype-db" +${GRYPE_DB} build + +step "Packaging grype-db" +${GRYPE_DB} package +GRYPE_DB_TAR=build/grype-db.tar.gz +mv build/vulnerability-db_*.tar.gz ${GRYPE_DB_TAR} + +step "Importing DB into grype" +${GRYPE} db import ${GRYPE_DB_TAR} diff --git a/.gitignore b/.gitignore index 9b75c5f1..1dd596de 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,15 @@ .vunnel.yaml +.grype-db.yaml +.grype.yaml +.grype +.grype-db + +/bin /data/ /backup/ .pytype/ .wily/ +.cache/ /.tmp/ CHANGELOG.md @@ -120,6 +127,7 @@ ENV/ .DS_Store .pytest_cache +.ruff_cache dropin.cache diff --git a/DEVELOPING.md b/DEVELOPING.md index c3e73d04..96fa05ae 100644 --- a/DEVELOPING.md +++ b/DEVELOPING.md @@ -6,16 +6,28 @@ This project requires: - python (>= 3.7) - pip (>= 22.2) - poetry (>= 1.2): see [installation instructions](https://python-poetry.org/docs/#installation) +- docker +- go (>= 1.20) +- posix shell (bash, zsh, etc... needed for the `make dev` "development shell") Once you have python and poetry installed, get the project bootstrapped: ```bash +# clone grype and grype-db, which is needed for provider development +git clone git@github.com:anchore/grype.git +git clone git@github.com:anchore/grype-db.git +# note: if you already have these repos cloned, you can skip this step. However, if they +# reside in a different directory than where the vunnel repo is, then you will need to +# set the `GRYPE_PATH` and/or `GRYPE_DB_PATH` environment variables for the development +# shell to function. You can add these to a local .env file in the vunnel repo root. + +# clone the vunnel repo +git clone git@github.com:anchore/vunnel.git +cd vunnel + # get basic project tooling make bootstrap -# get a persistent virtual environment to work within -poetry shell - # install project dependencies poetry install ``` @@ -30,25 +42,112 @@ To jump into a poetry-managed virtualenv run `poetry shell`, this will prevent t ## Developing -If you want to use a locally-editable copy of vunnel while you develop: +The easiest way to develop on a providers is to use the development shell, selecting the specific provider(s) you'd like to focus your development workflow on: ```bash -poetry shell -pip uninstall vunnel #... if you already have vunnel installed in this virtual env -pip install -e . +# Specify one or more providers you want to develop on. +# Any provider from the output of "vunnel list" is valid. +# Specify multiple as a space-delimited list: +# make dev providers="oracle wolfi nvd" +$ make dev provider="oracle" + +Entering vunnel development shell... +• Configuring with providers: oracle ... +• Writing grype config: /Users/wagoodman/code/vunnel/.grype.yaml ... +• Writing grype-db config: /Users/wagoodman/code/vunnel/.grype-db.yaml ... +• Activating poetry virtual env: /Users/wagoodman/Library/Caches/pypoetry/virtualenvs/vunnel-slR_vCr2-py3.9 ... +• Installing editable version of vunnel ... +• Building grype ... +• Building grype-db ... + +Note: development builds grype and grype-db are now available in your path. +To update these builds run 'make build-grype' and 'make build-grype-db' respectively. +To run your provider and update the grype database run 'make update-db'. +Type 'exit' to exit the development shell. +``` + +You can now run the provider you specified in the `make dev` command, build an isolated grype DB, and import the DB into grype: + +```bash +$ make update-db +• Updating vunnel providers ... +[0000] INFO grype-db version: ede464c2def9c085325e18ed319b36424d71180d-adhoc-build +... +[0000] INFO configured providers parallelism=1 providers=1 +[0000] DEBUG └── oracle +[0000] DEBUG all providers started, waiting for graceful completion... +[0000] INFO running vulnerability provider provider=oracle +[0000] DEBUG oracle: 2023-03-07 15:44:13 [INFO] running oracle provider +[0000] DEBUG oracle: 2023-03-07 15:44:13 [INFO] downloading ELSA from https://linux.oracle.com/security/oval/com.oracle.elsa-all.xml.bz2 +[0019] DEBUG oracle: 2023-03-07 15:44:31 [INFO] wrote 6298 entries +[0019] DEBUG oracle: 2023-03-07 15:44:31 [INFO] recording workspace state +• Building grype-db ... +[0000] INFO grype-db version: ede464c2def9c085325e18ed319b36424d71180d-adhoc-build +[0000] INFO reading all provider state +[0000] INFO building DB build-directory=./build providers=[oracle] schema=5 +• Packaging grype-db ... +[0000] INFO grype-db version: ede464c2def9c085325e18ed319b36424d71180d-adhoc-build +[0000] INFO packaging DB from="./build" for="https://toolbox-data.anchore.io/grype/databases" +[0000] INFO created DB archive path=build/vulnerability-db_v5_2023-03-07T20:44:13Z_405ae93d52ac4cde6606.tar.gz +• Importing DB into grype ... +Vulnerability database imported +``` + +You can now run grype that uses the newly created DB: + +```bash +$ grype oraclelinux:8.4 + ✔ Pulled image + ✔ Loaded image + ✔ Parsed image + ✔ Cataloged packages [195 packages] + ✔ Scanning image... [193 vulnerabilities] + ├── 0 critical, 25 high, 146 medium, 22 low, 0 negligible + └── 193 fixed + +NAME INSTALLED FIXED-IN TYPE VULNERABILITY SEVERITY +bind-export-libs 32:9.11.26-4.el8_4 32:9.11.26-6.el8 rpm ELSA-2021-4384 Medium +bind-export-libs 32:9.11.26-4.el8_4 32:9.11.36-3.el8 rpm ELSA-2022-2092 Medium +bind-export-libs 32:9.11.26-4.el8_4 32:9.11.36-3.el8_6.1 rpm ELSA-2022-6778 High +bind-export-libs 32:9.11.26-4.el8_4 32:9.11.36-5.el8 rpm ELSA-2022-7790 Medium + +# note that we're using the database we just built... +$ grype db status +Location: /Users/wagoodman/code/vunnel/.cache/grype/5 # <--- this is the local DB we just built +... + +# also note that we're using a development build of grype +$ which grype +/Users/wagoodman/code/vunnel/bin/grype ``` -To run all static-analysis and tests: +The development builds of grype and grype-db provided are derived from `../grype` and `../grype-db` paths relative to the vunnel project. +If you want to use a different path, you can set the `GRYPE_PATH` and `GRYPE_DB_PATH` environment variables. This can be +persisted by adding a `.env` file to the root of the vunnel project: ```bash -make +# example .env file in the root of the vunnel repo +GRYPE_PATH=~/somewhere/else/grype +GRYPE_DB_PATH=~/also/somewhere/else/grype-db ``` -Or run them individually: +To rebuild the grype and grype-db binaries from local source, run: ```bash -make static-analysis -make test +make build-grype +make build-grype-db +``` + +This project uses Make for running common development tasks: + +```bash + +make # run static analysis and unit testing +make static-analysis # run static analysis +make unit # run unit tests +make format # format the codebase with black +make lint-fix # attempt to automatically fix linting errors +... ``` If you want to see all of the things you can do: @@ -57,6 +156,186 @@ If you want to see all of the things you can do: make help ``` +If you want to use a locally-editable copy of vunnel while you develop without the custom development shell: + +```bash +poetry shell +pip uninstall vunnel #... if you already have vunnel installed in this virtual env +pip install -e . +``` + +## Architecture + +Vunnel is a CLI tool that downloads and processes vulnerability data from various sources (in the codebase, these are called "providers"). + + + + + +Conceptually, one or more invocations of Vunnel will produce a single data directory which Grype-DB uses to create a Grype database: + + + + + +Additionally, the Vunnel CLI tool is optimized to run +a single provider at a time, not orchestrating multiple providers at once. [Grype-db](github.com/anchore/grype-db) is the +tool that collates output from multiple providers and produces a single database, and is ultimately responsible for +orchestrating multiple Vunnel calls to prepare the input data: + + + + + +For more information about how Grype-DB uses Vunnel see [the Grype-DB documentation](https://github.com/anchore/grype-db/blob/main/DEVELOPING.md#architecture). + + +### Vunnel Providers + +A "Provider" is the core abstraction for Vunnel and represents a single source of vulnerability data. Vunnel is a CLI wrapper +around multiple vulnerability data providers. + +All provider implementations should... +- live under `src/vunnel/providers` in their own directory (e.g. the NVD provider code is under `src/vunnel/providers/nvd/...`) +- have a class that implements the [`Provider` interface](https://github.com/anchore/vunnel/blob/1285a3be0f24fd6472c1f469dd327541ff1fc01e/src/vunnel/provider.py#L73) +- be centrally registered with a unique name under [`src/vunnel/providers/__init__.py`](https://github.com/anchore/vunnel/blob/1285a3be0f24fd6472c1f469dd327541ff1fc01e/src/vunnel/providers/__init__.py) +- be independent from other vulnerability providers data --that is, the debian provider CANNOT reach into the NVD data provider directory to look up information (such as severity) +- follow the workspace conventions for downloaded provider inputs, produced results, and tracking of metadata + +Each provider has a "workspace" directory within the "vunnel root" directory (defaults to `./data`) named after the provider. + +```yaml +data/ # the "vunnel root" directory +└── alpine/ # the provider workspace directory + ├── input/ # any file that needs to be downloaded and referenced should be stored here + ├── results/ # schema-compliant vulnerability results (1 record per file) + ├── checksums # listing of result file checksums (xxh64 algorithm) + └── metadata.json # metadata about the input and result files +``` + +The `metadata.json` and `checksums` are written out after all results are written to `results/`. An example `metadata.json`: +```json +{ + "provider": "amazon", + "urls": [ + "https://alas.aws.amazon.com/AL2022/alas.rss" + ], + "listing": { + "digest": "dd3bb0f6c21f3936", + "path": "checksums", + "algorithm": "xxh64" + }, + "timestamp": "2023-01-01T21:20:57.504194+00:00", + "schema": { + "version": "1.0.0", + "url": "https://raw.githubusercontent.com/anchore/vunnel/main/schema/provider-workspace-state/schema-1.0.0.json" + } +} +``` +Where: +- `provider`: the name of the provider that generated the results +- `urls`: the URLs that were referenced to generate the results +- `listing`: the path to the `checksums` listing file that lists all of the results, the checksum of that file, and the algorithm used to checksum the file (and the same algorithm used for all contained checksums) +- `timestamp`: the point in time when the results were generated or last updated +- `schema`: the data shape that the current file conforms to + +All results from a provider are handled by a common base class helper (`provider.Provider.results_writer()`) and is driven +by the application configuration (e.g. JSON flat files or SQLite database). The data shape of the results are +self-describing via an envelope with a schema reference. For example: + +For example: +```json +{ + "schema": "https://raw.githubusercontent.com/anchore/vunnel/main/schema/vulnerability/os/schema-1.0.0.json", + "identifier": "3.3/cve-2015-8366", + "item": { + "Vulnerability": { + "Severity": "Unknown", + "NamespaceName": "alpine:3.3", + "FixedIn": [ + { + "VersionFormat": "apk", + "NamespaceName": "alpine:3.3", + "Name": "libraw", + "Version": "0.17.1-r0" + } + ], + "Link": "http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2015-8366", + "Description": "", + "Metadata": {}, + "Name": "CVE-2015-8366", + "CVSS": [] + } + } +} +``` + +Where: +- the `schema` field is a URL to the schema that describes the data shape of the `item` field +- the `identifier` field should have a unique identifier within the context of the provider results +- the `item` field is the actual vulnerability data, and the shape of this field is defined by the schema + +Note that the identifier is `3.3/cve-2015-8366` and not just `cve-2015-8366` in order to uniquely identify +`cve-2015-8366` as applied to the `alpine 3.3` distro version among other records in the results directory. + +Currently only JSON payloads are supported at this time. + +Possible vulnerability schemas supported within the vunnel repo are: +- [Generic OS Vulnerability](https://github.com/anchore/vunnel/tree/main/schema/vulnerability/os) +- [GitHub Security Advisories](https://github.com/anchore/vunnel/tree/main/schema/vulnerability/github-security-advisory) +- [NVD Vulnerability](https://github.com/anchore/vunnel/tree/main/schema/vulnerability/nvd) + + +### Provider configurations + +Each provider has a configuration object defined next to the provider class. This object is used in the vunnel application +configuration and is passed as input to the provider class. Take the debian provider configuration for example: + +```python +from dataclasses import dataclass, field + +from vunnel import provider, result + +@dataclass +class Config: + runtime: provider.RuntimeConfig = field( + default_factory=lambda: provider.RuntimeConfig( + result_store=result.StoreStrategy.SQLITE, + existing_results=provider.ResultStatePolicy.DELETE_BEFORE_WRITE, + ), + ) + request_timeout: int = 125 + +``` + +Every provider configuration must: +- be a `dataclass` +- have a `runtime` field that is a `provider.RuntimeConfig` field + +The `runtime` field is used to configure common behaviors of the provider that are enforced within the `vunnel.provider.Provider` subclass. Options include: + +- `on_error`: what to do when the provider fails, sub fields include: + - `action`: choose to `fail`, `skip`, or `retry` when the failure occurs + - `retry_count`: the number of times to retry the provider before failing (only applicable when `action` is `retry`) + - `retry_delay`: the number of seconds to wait between retries (only applicable when `action` is `retry`) + - `input`: what to do about the `input` data directory on failure (such as `keep` or `delete`) + - `results`: what to do about the `results` data directory on failure (such as `keep` or `delete`) + +- `existing_results`: what to do when the provider is run again and the results directory already exists. Options include: + - `delete-before-write`: delete the existing results just before writing the first processed (new) result + - `delete`: delete existing results before running the provider + - `keep`: keep the existing results + +- `existing_input`: what to do when the provider is run again and the input directory already exists. Options include: + - `delete`: delete the existing input before running the provider + - `keep`: keep the existing input + +- `result_store`: where to store the results. Options include: + - `sqlite`: store results as key-value form in a SQLite database, where keys are the record identifiers values are the json vulnerability records + - `flat-file`: store results in JSON files named after the record identifiers + +Any provider-specific config options can be added to the configuration object as needed (such as `request_timeout`, which is a common field). + ## Adding a new provider @@ -70,29 +349,60 @@ To add a new provider, you will need to create a new provider class under `/src - `update()`: downloads and processes the raw data, writing all results with `self.results_writer()` All results must conform to a [particular schema](https://github.com/anchore/vunnel/tree/main/schema), today there are a few kinds: -- `os`: a generic operating system vulnerability +- `os`: a generic operating system vulnerability (e.g redhat, debian, ubuntu, alpine, wolfi, etc.) - `nvd`: tailored to describe vulnerabilities from the NVD - `github-security-advisory`: tailored to describe vulnerabilities from GitHub +Once the provider is implemented, you will need to wire it up into the application in a couple places: +- add a new entry under the dispatch table in `src/vunnel/providers/__init__.py` mapping your provider name to the class +- add the provider configuration to the application configuration under `src/vunnel/cli/config.py` (specifically the `Providers` dataclass) + +For a more detailed example on the implementation details of a provider see the ["example" provider](example/README.md). + Validating this provider has different implications depending on what is being added. For example, if the provider is -adding a new vulnerability source but is ultimately using an existing schema to express results then there is very little to do! +adding a new vulnerability source but is ultimately using an existing schema to express results then there may be very little to do! If you are adding a new schema, then the downstream data pipeline will need to be altered to support reading data in the new schema. +**_Please feel free to reach out to a maintainer on an incomplete draft PR and we can help you get it over the finish line!_** + + ### ...for an existing schema -1. Fork Vunnel and add the new provider. +#### **1. Fork Vunnel and add the new provider.** + +Take a look at the example provider in the `example` directory. You are encouraged to copy `example/awesome/*` into +`src/vunnel/providers/YOURPROVIDERNAME/` and modify it to fit the needs of your new provider, however, this is not required: + +```bash +# from the root of the vunnel repo +cp -a example/awesome src/vunnel/providers/YOURPROVIDERNAME + +``` -You should be able to see the new provider in the `vunnel list` command and run it with `vunnel run `. +See the ["example" provider README](example/README.md) as well as the code comments for steps and considerations to take when implementing a new provider. + +Once implemented, you should be able to see the new provider in the `vunnel list` command and run it with `vunnel run `. The entries written should write out to a specific `namespace` in the DB downstream, as indicated in the record. This namespace is needed when making Grype changes. +While developing the provider consider using the `make dev provider=""`developer shell to run the provider and manually test the results against grype. + +_**At this point you can optionally open a Vunnel PR with your new provider and a Maintainer can help with the next steps.**_ Or if you'd like to get PR changes merged faster you can continue with the next steps. -2. Fork Grype and map distro type to a specific namespace. -This step might not be needed depending on the provider. Any change needed would be in the current grype db schema namespace index: https://github.com/anchore/grype/blob/main/grype/db/v5/namespace/index.go . +#### **2. Fork Grype and map distro type to a specific namespace.** +This step might not be needed depending on the provider. -3. In Vunnel: add a new test case to `tests/quality/config.yaml` for the new provider. +Common reasons for needing Grype changes include: +- Grype does not support the distro type and it needs to be added. See the [grype/distro/types.go](https://github.com/anchore/grype/blob/main/grype/distro/type.go) file to add the new distro. +- Grype supports the distro already, but matching is disabled. See the [grype/distro/distro.go](https://github.com/anchore/grype/blob/main/grype/distro/distro.go) file to enable the distro explicitly. +- There is a non-standard mapping of distro to namespaces (e.g. redhat and centos map to `rhel`). See the grype db schema namespace index for possible changes: https://github.com/anchore/grype/blob/main/grype/db/v5/namespace/index.go . + +If you're using the developer shell (`make dev ...`) then you can run `make build-grype` to get a build of grype with your changes. + + +#### **3. In Vunnel: add a new test case to `tests/quality/config.yaml` for the new provider.** The configuration maps a provider to test to specific images to test with, for example: ```yaml @@ -109,20 +419,21 @@ Always use both the image tag and digest for all container image entries. Pick an image that has a good representation of the package types that your new provider is adding vulnerability data for. -4. In Vunnel: swap the tools to your Grype branch in `tests/quality/config.yaml`. +#### **4. In Vunnel: swap the tools to your Grype branch in `tests/quality/config.yaml`.** If you wanted to see PR quality gate checks pass with your specific Grype changes (if you have any) then you can update the `yardstick.tools[*]` entries for grype to use the a version that points to your fork (w.g. `your-fork-username/grype@main`). If you don't have any grype changes needed then you can skip this step. -5. In Vunnel: add new "vulnerability match labels" to annotate True and False positive findings with Grype. +#### **5. In Vunnel: add new "vulnerability match labels" to annotate True and False positive findings with Grype.** In order to evaluate the quality of the new provider, we need to know what the expected results are. This is done by annotating Grype results with "True Positive" labels (good results) and "False Positive" labels (bad results). We'll use [Yardstick](github.com/anchore/yardstick) to do this: ```bash +# note: be certain you are in a "poetry shell" session $ cd tests/quality # capture results with the development version of grype (from your fork) @@ -150,9 +461,10 @@ Later we'll open a PR in the [vulnerability-match-labels repo](github.com/anchor For the meantime we can iterate locally with the labels we've added. -6. In Vunnel: run the quality gate. +#### **6. In Vunnel: run the quality gate.** ```bash +# note: be certain you are in a "poetry shell" session cd tests/quality # runs your specific provider to gather vulnerability data, builds a DB, and runs grype with the new DB @@ -167,7 +479,7 @@ This uses the latest Grype-DB release to build a DB and the specified Grype vers You are looking for a passing run before continuing further. -7. Open a [vulnerability-match-labels repo](github.com/anchore/vulnerability-match-labels) PR to persist the new labels. +#### **7. Open a [vulnerability-match-labels repo](github.com/anchore/vulnerability-match-labels) PR to persist the new labels.** Vunnel uses the labels in the vulnerability-Match-Labels repo via a git submodule. We've already added labels locally within this submodule in an earlier step. To persist these labels we need to push them to a fork and open a PR: @@ -202,7 +514,7 @@ $ git submodule update --remote vulnerability-match-labels ``` -8. In Vunnel: open a PR with your new provider. +#### **8. In Vunnel: open a PR with your new provider.** The PR will also run all of the same quality gate checks that you ran locally. @@ -219,66 +531,10 @@ This is the same process as listed above with a few additional steps: 4. The final Vunnel PR will not be able to be merged until the Grype-DB PR is merged and the `tests/quality/config.yaml` file is updated to point back to the `latest` Grype-DB version. -## Architecture - -Vunnel is a CLI tool that downloads and processes vulnerability data from various sources (in the codebase, these are called "providers"). -It is designed to be extensible and easy to add new providers. Additionally, the Vunnel CLI tool is optimized to run -a single provider at a time, not orchestrating multiple providers at once. [Grype-db](github.com/anchore/grype-db) is the -tool that collates output from multiple providers and produces a single database, and is ultimately responsible for -orchestrating multiple Vunnel calls to prepare the input data. - -All providers work within a common root directory, by default `./data`. Within this directory, each provider has its own -"workspace" subdirectory. By convention no provider should read or write outside of its own workspace. This implies that -providers are independent of each other and can be run in parallel safely. - -Within a provider's workspace, there are a few common subdirectories: -``` -data # vunnel root directory -└── wolfi # "wolfi" provider workspace - ├── input # contains all raw data downloaded by the provider - └── results # contains all processed data produced by the provider -``` - -All results from a provider are handled by a common base class helper (`provider.Provider.results_writer()`) and is driven -by the application configuration (e.g. JSON flat files or SQLite database). The data shape of the results are -self-describing via an envelope with a schema reference. For example: - -```json -{ - "schema": "https://raw.githubusercontent.com/anchore/vunnel/main/schema/vulnerability/os/schema-1.0.0.json", - "identifier": "wolfi:rolling/CVE-2007-2728", - "item": { - "Vulnerability": { - "Severity": "Unknown", - "NamespaceName": "wolfi:rolling", - "FixedIn": [ - { - "Name": "php", - "Version": "0", - "VersionFormat": "apk", - "NamespaceName": "wolfi:rolling" - } - ], - "Link": "http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2007-2728", - "Description": "The soap extension in PHP calls php_rand_r with an uninitialized seed variable, which has...", - "Metadata": {}, - "Name": "CVE-2007-2728", - "CVSS": [] - } - } -} -``` - -Note: -- the `schema` field is a URL to the schema that describes the data shape of the `item` field -- the `identifier` field should have a unique identifier within the context of the provider results -- the `item` field is the actual vulnerability data, and the shape of this field is defined by the schema - -Currently only JSON payloads are supported at this time. - - ## What might need refactoring? +Looking to help out with improving the code quality of Vunnel, but not sure where to start? + The best way is to look for [issues with the `refactor` label](https://github.com/anchore/vunnel/issues?q=is%3Aissue+is%3Aopen+label%3Arefactor). More general ways would be to use `radon` to search for complexity and maintainability issues: @@ -333,6 +589,7 @@ $ wily rank Ideally we should try to get `wily diff` output into the CI pipeline and post on a sticky PR comment to show regressions (and potentially fail the CI run). + ## Not everything has types... This codebase has been ported from another repo that did not have any type hints. This is OK, though ideally over time this should @@ -343,90 +600,3 @@ We use `mypy` today for static type checking, however, the ported code has been If you want to make enhancements in this area consider using automated tooling such as [`pytype`](https://github.com/google/pytype) to generate types via inference into `.pyi` files and later merge them into the codebase with [`merge-pyi`](https://github.com/google/pytype/tree/main/pytype/tools/merge_pyi). Alternatively a tool like [`MonkeyType`](https://github.com/Instagram/MonkeyType) can be used generate static types from runtime data and incorporate into the code. - -## Architecture - -Vunnel is a CLI wrapper around multiple vulnerability data providers. All provider implementations should... -- live under `src/vunnel/providers` in their own directory (e.g. the NVD provider code is under `src/vunnel/providers/nvd/...`) -- have a class that implements the [`Provider` interface](https://github.com/anchore/vunnel/blob/1285a3be0f24fd6472c1f469dd327541ff1fc01e/src/vunnel/provider.py#L73) -- be centrally registered with a unique name under [`src/vunnel/providers/__init__.py`](https://github.com/anchore/vunnel/blob/1285a3be0f24fd6472c1f469dd327541ff1fc01e/src/vunnel/providers/__init__.py) -- be independent from other vulnerability providers data --that is, the debian provider CANNOT reach into the NVD data provider directory to look up information (such as severity) -- follow the workspace conventions for downloaded provider inputs, produced results, and tracking of metadata - - -Each provider is given a "workspace" directory within the vunnel `root` directory named after the provider. - -```yaml -data/ # the "vunnel root" directory -└── alpine/ # the provider workspace directory - ├── input/ # any file that needs to be downloaded and referenced should be stored here - ├── results/ # schema-compliant vulnerability results (1 record per file) - ├── checksums # listing of result file checksums (xxh64 algorithm) - └── metadata.json # metadata about the input and result files -``` - -The `metadata.json` and `checksums` are written out after all results are written to `results/`. An example `metadata.json`: -```json -{ - "provider": "amazon", - "urls": [ - "https://alas.aws.amazon.com/AL2022/alas.rss" - ], - "listing": { - "digest": "dd3bb0f6c21f3936", - "path": "checksums", - "algorithm": "xxh64" - }, - "timestamp": "2023-01-01T21:20:57.504194+00:00", - "schema": { - "version": "1.0.0", - "url": "https://raw.githubusercontent.com/anchore/vunnel/main/schema/provider-workspace-state/schema-1.0.0.json" - } -} -``` -Where: -- `provider`: the name of the provider that generated the results -- `urls`: the URLs that were referenced to generate the results -- `listing`: the path to the `checksums` listing file that lists all of the results, the checksum of that file, and the algorithm used to checksum the file (and the same algorithm used for all contained checksums) -- `timestamp`: the point in time when the results were generated or last updated -- `schema`: the data shape that the current file conforms to - -All results stored in `results/**/*.json` should follow have `schema`, `identifier`, and `item` fields contained within an object. - -- `schema`: the vulnerability schema which the `.item` field conforms to -- `identifier`: a string that uniquely identifies the current vulnerability record within the entire `results` directory -- `item`: the vulnerability record - -For example: -```json -{ - "schema": "https://raw.githubusercontent.com/anchore/vunnel/main/schema/vulnerability/os/schema-1.0.0.json", - "identifier": "3.3/cve-2015-8366", - "item": { - "Vulnerability": { - "Severity": "Unknown", - "NamespaceName": "alpine:3.3", - "FixedIn": [ - { - "VersionFormat": "apk", - "NamespaceName": "alpine:3.3", - "Name": "libraw", - "Version": "0.17.1-r0" - } - ], - "Link": "http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2015-8366", - "Description": "", - "Metadata": {}, - "Name": "CVE-2015-8366", - "CVSS": [] - } - } -} -``` - -Note that the identifier is `3.3/cve-2015-8366` and not just `cve-2015-8366` in order to uniquely identify `cve-2015-8366` as applied to the `alpine 3.3` distro version among other records in the results directory. - -Possible vulnerability schemas supported within the vunnel repo are: -- [GitHub Security Advisories](https://github.com/anchore/vunnel/tree/main/schema/vulnerability/github-security-advisory) -- [Generic OS Vulnerability](https://github.com/anchore/vunnel/tree/main/schema/vulnerability/os) -- [NVD Vulnerability](https://github.com/anchore/vunnel/tree/main/schema/vulnerability/nvd) diff --git a/Makefile b/Makefile index 1c3ae502..ddcff511 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,11 @@ TEMP_DIR = ./.tmp +BIN_DIR = ./bin +ABS_BIN_DIR = $(shell realpath $(BIN_DIR)) + +# path to the grype repo, defaults to ../grype if not set in the GRYPE_PATH environment variable (same for the grype-db repo) +GRYPE_PATH ?= ../grype +GRYPE_DB_PATH ?= ../grype-db + CRANE = $(TEMP_DIR)/crane CHRONICLE = $(TEMP_DIR)/chronicle GLOW = $(TEMP_DIR)/glow @@ -35,6 +42,26 @@ endif .PHONY: all all: static-analysis test ## Run all validations +.PHONY: dev +dev: ## Get a development shell with locally editable grype, grype-db, and vunnel repos + @DEV_VUNNEL_BIN_DIR=$(ABS_BIN_DIR) .github/scripts/dev-shell.sh $(provider) $(providers) + +.PHONY: build-grype +build-grype: $(TEMP_DIR) ## Build grype for local development + @cd $(GRYPE_PATH) && go build -o $(ABS_BIN_DIR)/grype . + +.PHONY: build-grype-db +build-grype-db: $(TEMP_DIR) ## Build grype-db for local development + @cd $(GRYPE_DB_PATH) && go build -o $(ABS_BIN_DIR)/grype-db ./cmd/grype-db + +.PHONY: update-db +update-db: check-dev-shell ## Build and import a grype database based off of the current configuration + @.github/scripts/update-dev-db.sh + +.PHONY: check-dev-shell +check-dev-shell: + @test -n "$$DEV_VUNNEL_SHELL" || (echo "$(RED)DEV_VUNNEL_SHELL is not set. Run 'make dev provider=\"...\"' first$(RESET)" && exit 1) + $(TEMP_DIR): mkdir -p $(TEMP_DIR) diff --git a/docs/grype-db-actions.drawio b/docs/grype-db-actions.drawio new file mode 100644 index 00000000..5906ec3b --- /dev/null +++ b/docs/grype-db-actions.drawio @@ -0,0 +1 @@ +7Vzbdto4FP0a1pp5gOU75jEJSdppeplJZ9p5lG1hPBEWlWUC/fqRbMn4RoBiAyF+CdaxkGWdffa5SKGn38yW9wTMpx+xB1FPU7xlTx/3NE1VVJt9cMlKSHTNSiU+CTwhWwseg59QflVI48CDUaEjxRjRYF4UujgMoUsLMkAIfi52m2BUfOoc+LAieHQBqkq/BR6dplJbG67l72DgT+WTVWuU3pkB2Vm8STQFHn7OifTbnn5DMKbp1Wx5AxFfPbku396vvqGHJ+v+jz+jH+Dv6w9fP/3TTwe72+cr2SsQGNJfHjoI4Yf4++Tq4cZ3P/fff9Q/Le/6thh7AVAsFky8LF3JFYQeW1DRhMjBz7drwTXBcehB/giFtTChU+zjEKAHjOdMqDLhf5DSlQAGiClmoimdIXF3gkN6B2YB4gB7B9EC0sAF4ob4lsrW+pq9O1l9Fw9KGv/yxsCUzfEyf3O8yre+QBLMIIVECHdcUbE6EY6JC19YRktYCQXEh/Sl9RZY4muaw6fQ2D3EbJJkxToQiAANFkUQA2ELftZPfPWKELDKdZjjIKRRbuQvXMA6CLs2R2JEadVmATvsIh1RtnJTW4sSfO2BNblIBaxZiApdc5IBFPT5dT8ibtLD+hFz82KAoZwvrvgUtTveJRr4GPsIgnkQDVw8Y2I3Yl3uJhJMf2EHM7Rp1x9xiNdDcRIBQptSJrsq5a6Wzz8XMeMm/n4kDvkqozl7QTl3MR/Zt2I+RQt5ngYUPs7TCTwzzt3RGhaQ8Ct0hQI/ZPdmgeclBgiEAMEJf3zEhg5C/yFpjbltTAKEbjDCJJmO7gFoT1zekxL8BHN3LNeGzmRv4+Azg8sXwSzuGpJJJepk+3lNzKphpLJpjpRlvzr85zC7PyRtraO/BuhvqLxy+rOOQH9ykV49/ZEpb3TkdyD5mScnP70jvybI77XHfkPjCOR3KbFfuPA67juY++yTc5/RcV8T3Dd63dynKccI/EYXwn2DwaDjvkO5LyuMno77tiYia0VKRKTgTFiJYVFRtfkyxckaXOcKYobZOz439rEHgKMpmPNLd4UChmSib4exk2L+wckEwH3yE0v4HFPES0apPBLsbu7qBnhHBByIvuAooAHm0HcZTjmtZ0byUOpAuSMqW0rZotjKUbb87cHfKrt+Y1iBv8yE8ui3WkN/XSR6Os9fBK9Qu3S7qeFoJjcdkxuPucl8zNSAWFdpQuwyG9wUZlQTXPC3Yw2Ji+ZDBNvcNUQYNh0iHIaTqtcmEHi1WElMr6hkaXcVM634MgIZHwAnGY8jSsQwbHDzumeOa5S2kTOqqnvZBMpWnO3Cien08htdddbdVwaGpZoFC++LssJ+kVwlVFON4qjyKXIEPJlEDE9lNTcQr0nE5jQ/J3gRMC+QerHUgzhk7TwsMONkGzrRPHVS/V55m2Br93xZbWvnXBq6tW/i9Ta4Nw+78SxBy97ebTuN5SArXJ4yUE3d5pCRf83tPip1ZDUxXrtU2ZZHtHfYBFFqPKKqtuYSrY0BIVv1MAukfLKaw77nDFZglq8A5ztVkMZWhRaRVIzBQ8zjokLALkRbYJERaB1wi076RQKtha6Y/64774eCYFSDAb0GA3pbEBjV5QSXVQ9pOq4Z7hjXyNLnmcQ1sgJyNMX+qr52BsSpFGudmWKPbbEXq9hh48XKwxSrd4ptSLGj81JsdRMCLqEbU1ir8FeWZUrYNpJlWopuF8PmXhNJplUYs2+UBmgxxxxuDLkdGUrfJ+H2+DoXaTtHLrjX1hC3V9tNaHtGJdJnd2zN0S2rxeRKM0swqc2u1GpkbbcWWVerCXtX2+1KtT1L+A8d6GzL9jLZ5DCJ0eYDR7uV888+C624y7YzUKOYgRpVO9GGx8xAVaXOULrC/LowD5cB/S5DNHadC8pYax2T8cYq1yifATiTBFpVrR3DNlXWghs+PVCtOZfKMkZ5DzZ9KfGt5qMCValW4i5l02Ft392uQz3Iu41JATe7DbIxdiab89qHzGbepQrtpgr68NSpgqp2ucKhuYITB8jrkoV2kwVdOXm2oG7es+wOsXWH2JqFfxa7SvzXnF8/6iG2LEh5I/BfxCiEBDgBCuhqwIm+fPblt+gHuwd/rz0V8yaNZWMSmJlLKRabTCaaW3tI2rMcy2wzFjPKDub0BlZ3aP9NZmNvpBql7bqJqEoktF6NMotexzKOXI3SqoeCLqYaldl3V42q1/15/TDMhVWjjkUhZb9aoZCU7VqkEG1jnNoVrxosXlk1GfmRi1da3c8JdMWrvTa6WfrAf86sK1+1Wr4yaw6FHLl8pXX5e5e/X07+nv3DinRHNf/kfNz8XdthNzVDTOxsYrVfAYsHyNNnHjvQJFcdKGZRqCXScwtxJcSVYyC3NaIvx7s1TG/U4LAcFjeHQ72aQxfIkDn+/vjq6+2AReID/2cLznx7+t3wP1C1pVy9VA8x5fnpvBuXR3oPdOOsuf5h1zQbWv8+rn77Pw== diff --git a/docs/vunnel+grype-db-workflow.drawio b/docs/vunnel+grype-db-workflow.drawio new file mode 100644 index 00000000..e2d7453c --- /dev/null +++ b/docs/vunnel+grype-db-workflow.drawio @@ -0,0 +1 @@ +7Ztbd5s4EMc/jc/ZfbCPAUOcx9i5tFunTZvdTfZRgMDayIgI+dZPvxKSuATskDbUzoaXGEZiEJq/fgMD6VnTxeaKgnh+TXyIe+bQ3/Ss855pGkNjzH+EZasslulIS0iRr2y54RZ9h/pQZV0iHyaljowQzFBcNnokiqDHSjZAKVmXuwUEl88agxBWDLcewFXrHfLZXFrH5klu/wBRONdnNpxT2bIAurO6kmQOfLIumKyLnjWlhDC5tdhMIRazp+fl7uP2Ds8enKs/viaP4K/Jpz8//92Xzi5fckh2CRRG7Iddowh+Wt4HZ7Np6H3pf7y2Pm8u+7Ypfa8AXqoJUxfLtnoGoc8nVO3yASC2/QYxYIhEF3nLhJJl5ENxriHfI5TNSUgigGeExNxocOO/kLGtUghYMsJNc7bAqjUgEbsEC4SF0r4Rl/AO5vCaREQ13pIl9cSxc8a4eEzbOuN/+PWKP6JDMggJCTEEMUoGHlmkDV6Sdr0MpGu+mTm3zUnRvRyYweM6aTjhKjCJHtiuWbaU2BigIWT7wqE8iikvyFcF9AqSBWSUX8SQphFYlTUO1FIJs365HPiGUsRL1GHVqMPBTM2YWI5AXbjzuBTrYPIB4hVkyAO5iW+F4ndNEYP6+J4MmW6rFd0MuJxGJZEAjMKIb3s8MJBywwpScTZ8phoWyPelHmGCvgM39ScUGRPEFSKc25OefV4T9L0CrCpi/3riw4KbXg3g1JBKDCmFVh3VHw6sE1seqtir9dY4+Mr5jbj0ohunfAgJgoTL8qlasjH9uIC08GsE5OrQX9FtDPvnk4Iy3J2yKFNmPeeKuo2lBtc8ge0hSi7MqmiYQFSmLQwDce6E+0VROEv3zm3hD2E8JZjQdCyWb8OxPxI9GSUPsNAyNl3LcV7MkYpodopjZJ6WlGGcqBy1zlMcz93SNi+kt9NhS6gYjbtE8gsSyWnDRKIX3pEkEj3u3Ykkl4lOG6Khn6RzycMzNMx4I5NJllaGPmCgL/tRr3SwjKw48LK3K6ZpRLN45tGUsSyep5LlyqqqZLoBdy7Gxn8AjvmM8I3BYNA09/EbzVhseluM+FKg1vOoc+WimbmZAXgPYbqUviwZFkOQ9kSJ097Hx6KKRUcsEvENSZBYq3XZd/akQy1Qn1KXTyHjcWiPkqamoqLkqApJe1hlpNMaI+tWQcfIV2akjvJbY6Qe9/tjJJ2LokNHyMMT0rDswyLSrlsEHSJfG5G6EPfWEGnUqONdIDJa+R0hj4GQpnXom0j7ZYTMydfRsSEdzaZ0PD0uOtbV8kt0PFbQrZZRlL53ostIzIx6Zm5GunZKglkduVJxfr4uCOA48Orqgo43hm7QZl2wDCu7ejdnjEZVWFmtwcrpYNUyrKyGsNLSOBZYPftq6Y3ASj68dqj6SVQZds0bjF/Lqrp3VR2rXpNVo6asOq7X4Hrcb55V6VNkh6qfRJU5OvRtlf2+6iCrJY4gBS7CiG0Hvpt9QkB1j9+SR94Gf6+2vNd6yc4PZLKKyZP1FQSB6dWuL99xHbvNrxkc48m9wPjQVeiTvZm/xaIz3CB2X9j+R7ga2GrvfKM8pzvbws4NpIhfu4i4tEV8Hu6LOwVPYjd3le5pX43k2cKtgd301uDIKtL2s7cG/ysSFyrSaS26q0gfviJtHboibXcV6bbp6DSko8bRsdCxrvz3Fh+cXoC57sFp54OT1eKDk/iMJPtPDPn5cv4PLdbFfw== diff --git a/docs/vunnel-run-workflow.drawio b/docs/vunnel-run-workflow.drawio new file mode 100644 index 00000000..7d63c37e --- /dev/null +++ b/docs/vunnel-run-workflow.drawio @@ -0,0 +1 @@ +7Ztdf6I6EIc/jZfy4y0WL2tbt3vW7vZsz2l3LyNEzGkkbgi+7Kc/CQQUglW3aq31ZpeMYYDMfx5ghjacq9HsE4Pj4R0NEGnYZjBrONcN27ZMyxP/Scs8s9gA2JklZDhQsxaGB/wb5bsqa4IDFJcmckoJx+Oy0adRhHxeskHG6LQ8bUBJ+ahjGCLN8OBDolufcMCHmdWzLxb2W4TDYX5kq9XOfhnBfLK6kngIAzpdMjk3DeeKUcqzrdHsChG5evm6PH2eP5Hec+vTX3/Hv+C/nS//fH1sZs662+xSXAJDEf9j1zhCX5Ifg8veVeh/a36+c77Ouk3LUc4nkCRqxRp2i4jDdPpiI5Qbl2QsdhaTHpAv/r3uFDNYPiW3iMP3q7YBFWctFAQ5bMrtZsz89EitX4lcu86QcymGS3nudldOiY2Q0pAgOMax4dORMPuxmNIdwBEmUorfaZ9y2rA7dzSiC1dSIdBHJff5VLM6NTvNGPlB34DpNRIcJTODstAwjKVLyq4gv6pUC3yeC0ysstAykp6GcCyNPqGJWPTOdIg5ehhn5zMV+ZVe64iIkVV4WhvVPECIcTRb0rSK8idER4izuZgyK2eeytimZeWW6SIBbEfZhkvibykbVDkXFr4XuhIbSlpbycxeL7Ovj9dSXUImJ6YvNsE+io1oEhgRjvkpaat9BNJy1kvrOwpuoRTJY0IiQ4nlpFTGUDCEXB1FIC1hmM9PSWniUeTtpeaul9pjIp5jSL2SKmvOaBIFSB7RXL/OMnrdXDe3iEwQxz4UP8glFVvkkuAwEr9xKveEakTQIIWQ8IujsJeOroH0hwm5ooSy9FycACJv4MuZnNFntPRLy/dQf7DHSNttp3rDaumhtto1oXbN/cUa6DkSiCdLNaSMD2lII0huFtZOOaKLOT0qg5LG8T/E+Vw9JsNEZHQpymiG+Y+l7Z/SlQHU6HqmPKeDeT6IxAX/WB78zD3IwWKndCT3alogH98jhsWKIaYmlWRWZk724wNNWCrRjHc2kMQDknlgFfVAxj0xNSef2CycA0U/5T5bGcveWnBxfmKrQ6qe+TlkIeIvzXRb2UwZ8RclzBCBHE/K7xp7EGNLA89UAB7VSrQH+wJAJVnlNPDFSspQa9AY4SDIFIxi/Bv2U39SD2OKRUilc9BpgOuaKK0Ekx6+lzNNI0bxUqjOp/TeVUeSpmlYJZQosGwcJeX5Xl700q2o7LR5UXZAB4NYyKka5OL0XhP3Cy3uxcvYPaMT8WrN9nRb2RftHbtKe7MNdNrbNbR39gd7733DXu21AvfmR4S9a24M+4sjg337lGHv7RL2oIwSdze4dyuAcg/G+1y0S6H/fnvTOyHat98e9q6lLeL7gv0LqP+YD/auszHrveNivavXRU+H9e4qXGzNetMw7QrrrZ2wvor6ioN9ol4vW2YF8FMhvXcEpF9dr8OltV2qplbqqjAIMMeS9AWJBPDM5dpxjZ+sQJzmkoCoadnjWa33perhRmdTKUavP/DFigO/VBleXQMXNryymil0wssaLJcSIxqhSt1RmTaHWJ3oy2mxUvevuwdtngegUrf27JpaJjhsGoDdpsFYESp+L1kAzllw4CxwrOrd4AjSQC+ibq0kXcJH2xEU0u5mbc4uVDXD7hYNwaINOCdYKIs56585+pkGe/3CAP3nMFXmt4QTeQoqJZQCwaZSlROJfPq9p3HKobo86VUm1DbCqkkllpCLOOzxflC9IViWrT8Y1WXC/vqYrl5W/iCZwIayN3vOg2PIg7qG/oHzwPuoeRBNgnMaHEUa1L0mHzgL9H6DFv4DFEiPqUCZF2tNw7vIS7QbFWzFoFp43U5JG3zFkH9FvbbY6ey8sVVfRLNzBS3Q7lXUml2W2m8P1TSgN04qJNdguYCLBsqATiNCYbDxx3NvXJytZosuuTWZv4tmnOsBpbfX9t+8qprKHvZYlAV1X8OfUXi0KNy872O+EQrbByeh3lFa4OzIuPUnTSWws6aSYJbj2tVb124QZle8Hq6vBOo+hz8j7GgRZm+MMOeNEOYdHGF62+60EJan6E6+gdIQ9sqPXgtmOYdj1o4bVFPKnuVfPKBzh+rcoVpViHEO2KiVlefi74SztFn8ubVz8z8= diff --git a/example/.gitignore b/example/.gitignore new file mode 100644 index 00000000..1269488f --- /dev/null +++ b/example/.gitignore @@ -0,0 +1 @@ +data diff --git a/example/README.md b/example/README.md new file mode 100644 index 00000000..0779734a --- /dev/null +++ b/example/README.md @@ -0,0 +1,47 @@ +# Example "awesome" provider + +This is an example of a provider for the "awesome" vulnerability data source. When writing a new provider it is +encouraged to copy this example (within the `awesome` directory) and modify it to fit the needs of the new provider. + +Take note of the `NOTE: CHANGE ME!` comments in the code. These are places where you will require tailoring. + +If you wanted to run the provider locally to try it out independent of the vunnel CLI: + +```bash +# run from the example/ directory + +poetry run python run.py +``` + +Then checkout the `./data` directory to see what was created. + + +## Code organization + +There tend to be two files in a directory that make up a provider: +```bash +src/vunnel/providers// + ├── __init__.py + └── parser.py +``` + +Where: +- `__init__.py` defines: + - defining the `Provider` class, which subclasses `vunnel.provider.Provider`. Try to keep this class as small as possible, importing functions and helpers from the surrounding `*.py` files. + - defining the provider `Config` class. This is used in the vunnel application configuration to define the configuration for the provider. +- `parser.py` is where any downloading and parsing logic lives. Feel free to change this name based off of the specific needs and implementation that you need. + + +## Considerations for writing a provider + +1. If possible, logically split up the downloading and parsing logic into separate functions. This will make it easier to test and debug. +2. Each provider may have different runtime configuration needs, tailor the specific defaults of the provider configuration you create accordingly +3. Pay careful attention to the runtime option for `existing_input`, if your provider requires previous data to accumulate over time, forbid customizing this value via configuration. +4. Make certain in the provider constructor to validate if the configuration is valid. If any fields are missing or have improper values raise a `ValueError` to prevent creation of the provider object. + + +## Testing your provider + +Unit tests for providers can be found under `tests/unit/providers//`. Tests should at least cover: +- parsing logic from static test fixture data representing downloaded data +- a `test_provider_schema` test that validates the provider is always outputting data in the correct schema diff --git a/example/awesome/__init__.py b/example/awesome/__init__.py new file mode 100644 index 00000000..206ff740 --- /dev/null +++ b/example/awesome/__init__.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING + +from vunnel import provider, result, schema + +from .parser import Parser + +if TYPE_CHECKING: + import datetime + +# NOTE, CHANGE ME!: a unique and semantically useful name for this provider +PROVIDER_NAME = "my-awesome-provider" + +# NOTE, CHANGE ME!: the data shape that all entries produced by this provider conform to +SCHEMA = schema.OSSchema() + + +@dataclass +class Config: + runtime: provider.RuntimeConfig = field( + default_factory=lambda: provider.RuntimeConfig( + result_store=result.StoreStrategy.SQLITE, + existing_results=provider.ResultStatePolicy.DELETE_BEFORE_WRITE, + ), + ) + request_timeout: int = 125 + + # NOTE, CHANGE ME!: Example for fetching secrets from the environment and sanitizing output. + # It is important to sanitize the __str__ method so that these secrets are not accidentally + # written to log output. + # + # token: str = "env:VUNNEL_AWESOME_TOKEN" + # + # def __post_init__(self) -> None: + # if self.token.startswith("env:"): + # self.token = os.environ.get(self.token[4:], "") + # + # def __str__(self) -> str: + # # sanitize secrets from any output + # tok_value = self.token + # str_value = super().__str__() + # if not tok_value: + # return str_value + # return str_value.replace(tok_value, "********") + + +class Provider(provider.Provider): + def __init__(self, root: str, config: Config | None = None): + if not config: + config = Config() + + super().__init__(root, runtime_cfg=config.runtime) + self.config = config + self.logger.debug(f"config: {config}") + + self.parser = Parser( + ws=self.workspace, + download_timeout=self.config.request_timeout, + logger=self.logger, + # NOTE, CHANGE ME!: example of passing a config secret to the parser to download the vulnerability data + # token=self.config.token + ) + + # this provider requires the previous state from former runs + provider.disallow_existing_input_policy(config.runtime) + + @classmethod + def name(cls) -> str: + return PROVIDER_NAME + + def update(self, last_updated: datetime.datetime | None) -> tuple[list[str], int]: + + # NOTE: CHANGE ME! Why is last_updated passed in here? This allows you to be able to make decisions about + # incremental updates of the existing vulnerability data state instead of needing to download all + # vulnerability data from the source. For an example of this see the NVD provider implementation at + # https://github.com/anchore/vunnel/blob/main/src/vunnel/providers/nvd/manager.py + + with self.results_writer() as writer: + + for vuln_id, record in self.parser.get(): + vuln_id = vuln_id.lower() + + writer.write( + identifier=vuln_id, + schema=SCHEMA, + payload=record, + ) + + return self.parser.urls, len(writer) diff --git a/example/awesome/parser.py b/example/awesome/parser.py new file mode 100644 index 00000000..4958d016 --- /dev/null +++ b/example/awesome/parser.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +import json +import logging +import os + +import requests +from vunnel import utils, workspace + +# NOTE, CHANGE ME!: this namespace should be unique to your provider and match expectations from +# grype to know what to search for in the DB. +NAMESPACE = "GRYPEOSNAMESPACETHATYOUCHOOSE" + + +class Parser: + # NOTE, CHANGE ME!: remove / add / change these attributes as needed to download and parse your provider + _json_url_ = "https://services.nvd.nist.gov/made-up-location" + _json_file_ = "vulnerability_data.json" + + def __init__(self, ws: workspace.Workspace, download_timeout: int = 125, logger: logging.Logger | None = None): + self.workspace = ws + self.download_timeout = download_timeout + self.json_file_path = os.path.join(ws.input_path, self._json_file_) + + # NOTE, CHANGE ME!: you should always record any URLs accessed in this list, either + # statically or dynamically within _download() + self.urls = [self._json_url_] + + if not logger: + logger = logging.getLogger(self.__class__.__name__) + self.logger = logger + + def get(self): + self._download() + yield from self._normalize() + + @utils.retry_with_backoff() + def _download(self): + self.logger.info(f"downloading vulnerability data from {self._json_url_}") + + r = requests.get(self._json_url_, timeout=self.download_timeout) + r.raise_for_status() + + with open(self.json_file_path, "w", encoding="utf-8") as f: + f.write(r.text) + + def _normalize(self): + + with open(self.json_file_path, encoding="utf-8") as f: + + for input_record in json.loads(f.read()): + + vuln_id = input_record["name"] + + # NOTE: this is in the data shape described by the OS vulnerability schema + yield vuln_id, { + "Vulnerability": { + "Name": vuln_id, + "NamespaceName": NAMESPACE, + "Link": f"https://someplace.com/{vuln_id}", + "Severity": input_record["severity"], + "Description": input_record["description"], + "FixedIn": [ + { + "Name": p, + "VersionFormat": "apk", + "NamespaceName": NAMESPACE, + "Version": input_record["fixed"] or "None", + } + for p in input_record["packages"] + ], + }, + } diff --git a/example/run.py b/example/run.py new file mode 100644 index 00000000..75cd9c31 --- /dev/null +++ b/example/run.py @@ -0,0 +1,60 @@ +# ruff: noqa: INP001 + +import json +import logging +from unittest import mock + +import awesome +from vunnel import provider, result + +fakedata = [ + { + "name": "FAKE-SA-001", + "packages": ["curl"], + "severity": "Critical", + "description": "Bad thing, really bad thing", + "affected": "1.0", + "fixed": "2.0", + }, + { + "name": "FAKE-SA-002", + "packages": ["wget"], + "severity": "Low", + "description": "Not really a bad thing, but no fix yet", + "affected": "5.0", + "fixed": None, + }, +] + + +def main(): + # we just want to show the parser working, but not against any real data. For that reason we'll mock + # this like we would in a unit test. + with mock.patch("awesome.parser.requests.get") as get: + get.return_value = mock.Mock( + json=lambda: fakedata, + text=json.dumps(fakedata), + raise_for_status=lambda: None, + status_code=200, + ) + + config = awesome.Config( + runtime=provider.RuntimeConfig( + result_store=result.StoreStrategy.FLAT_FILE, + existing_results=provider.ResultStatePolicy.DELETE_BEFORE_WRITE, + ), + ) + + my_provider = awesome.Provider(root="./data", config=config) + my_provider.run() + + +if __name__ == "__main__": + + logging.basicConfig( + level=logging.DEBUG, + format="[%(levelname)s] %(message)s", + handlers=[logging.StreamHandler()], + ) + + main() diff --git a/poetry.lock b/poetry.lock index 8f0e1180..ebb8b190 100644 --- a/poetry.lock +++ b/poetry.lock @@ -81,100 +81,87 @@ files = [ [[package]] name = "charset-normalizer" -version = "3.0.1" +version = "3.1.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." category = "main" optional = false -python-versions = "*" +python-versions = ">=3.7.0" files = [ - {file = "charset-normalizer-3.0.1.tar.gz", hash = "sha256:ebea339af930f8ca5d7a699b921106c6e29c617fe9606fa7baa043c1cdae326f"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88600c72ef7587fe1708fd242b385b6ed4b8904976d5da0893e31df8b3480cb6"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c75ffc45f25324e68ab238cb4b5c0a38cd1c3d7f1fb1f72b5541de469e2247db"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:db72b07027db150f468fbada4d85b3b2729a3db39178abf5c543b784c1254539"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:62595ab75873d50d57323a91dd03e6966eb79c41fa834b7a1661ed043b2d404d"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ff6f3db31555657f3163b15a6b7c6938d08df7adbfc9dd13d9d19edad678f1e8"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:772b87914ff1152b92a197ef4ea40efe27a378606c39446ded52c8f80f79702e"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70990b9c51340e4044cfc394a81f614f3f90d41397104d226f21e66de668730d"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:292d5e8ba896bbfd6334b096e34bffb56161c81408d6d036a7dfa6929cff8783"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:2edb64ee7bf1ed524a1da60cdcd2e1f6e2b4f66ef7c077680739f1641f62f555"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:31a9ddf4718d10ae04d9b18801bd776693487cbb57d74cc3458a7673f6f34639"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:44ba614de5361b3e5278e1241fda3dc1838deed864b50a10d7ce92983797fa76"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:12db3b2c533c23ab812c2b25934f60383361f8a376ae272665f8e48b88e8e1c6"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c512accbd6ff0270939b9ac214b84fb5ada5f0409c44298361b2f5e13f9aed9e"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-win32.whl", hash = "sha256:502218f52498a36d6bf5ea77081844017bf7982cdbe521ad85e64cabee1b608b"}, - {file = "charset_normalizer-3.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:601f36512f9e28f029d9481bdaf8e89e5148ac5d89cffd3b05cd533eeb423b59"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0298eafff88c99982a4cf66ba2efa1128e4ddaca0b05eec4c456bbc7db691d8d"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a8d0fc946c784ff7f7c3742310cc8a57c5c6dc31631269876a88b809dbeff3d3"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:87701167f2a5c930b403e9756fab1d31d4d4da52856143b609e30a1ce7160f3c"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:14e76c0f23218b8f46c4d87018ca2e441535aed3632ca134b10239dfb6dadd6b"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0c0a590235ccd933d9892c627dec5bc7511ce6ad6c1011fdf5b11363022746c1"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c7fe7afa480e3e82eed58e0ca89f751cd14d767638e2550c77a92a9e749c317"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79909e27e8e4fcc9db4addea88aa63f6423ebb171db091fb4373e3312cb6d603"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8ac7b6a045b814cf0c47f3623d21ebd88b3e8cf216a14790b455ea7ff0135d18"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:72966d1b297c741541ca8cf1223ff262a6febe52481af742036a0b296e35fa5a"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:f9d0c5c045a3ca9bedfc35dca8526798eb91a07aa7a2c0fee134c6c6f321cbd7"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:5995f0164fa7df59db4746112fec3f49c461dd6b31b841873443bdb077c13cfc"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4a8fcf28c05c1f6d7e177a9a46a1c52798bfe2ad80681d275b10dcf317deaf0b"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:761e8904c07ad053d285670f36dd94e1b6ab7f16ce62b9805c475b7aa1cffde6"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-win32.whl", hash = "sha256:71140351489970dfe5e60fc621ada3e0f41104a5eddaca47a7acb3c1b851d6d3"}, - {file = "charset_normalizer-3.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:9ab77acb98eba3fd2a85cd160851816bfce6871d944d885febf012713f06659c"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:84c3990934bae40ea69a82034912ffe5a62c60bbf6ec5bc9691419641d7d5c9a"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74292fc76c905c0ef095fe11e188a32ebd03bc38f3f3e9bcb85e4e6db177b7ea"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c95a03c79bbe30eec3ec2b7f076074f4281526724c8685a42872974ef4d36b72"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f4c39b0e3eac288fedc2b43055cfc2ca7a60362d0e5e87a637beac5d801ef478"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:df2c707231459e8a4028eabcd3cfc827befd635b3ef72eada84ab13b52e1574d"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:93ad6d87ac18e2a90b0fe89df7c65263b9a99a0eb98f0a3d2e079f12a0735837"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:59e5686dd847347e55dffcc191a96622f016bc0ad89105e24c14e0d6305acbc6"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:cd6056167405314a4dc3c173943f11249fa0f1b204f8b51ed4bde1a9cd1834dc"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:083c8d17153ecb403e5e1eb76a7ef4babfc2c48d58899c98fcaa04833e7a2f9a"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:f5057856d21e7586765171eac8b9fc3f7d44ef39425f85dbcccb13b3ebea806c"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:7eb33a30d75562222b64f569c642ff3dc6689e09adda43a082208397f016c39a"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-win32.whl", hash = "sha256:95dea361dd73757c6f1c0a1480ac499952c16ac83f7f5f4f84f0658a01b8ef41"}, - {file = "charset_normalizer-3.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:eaa379fcd227ca235d04152ca6704c7cb55564116f8bc52545ff357628e10602"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:3e45867f1f2ab0711d60c6c71746ac53537f1684baa699f4f668d4c6f6ce8e14"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cadaeaba78750d58d3cc6ac4d1fd867da6fc73c88156b7a3212a3cd4819d679d"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:911d8a40b2bef5b8bbae2e36a0b103f142ac53557ab421dc16ac4aafee6f53dc"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:503e65837c71b875ecdd733877d852adbc465bd82c768a067badd953bf1bc5a3"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a60332922359f920193b1d4826953c507a877b523b2395ad7bc716ddd386d866"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:16a8663d6e281208d78806dbe14ee9903715361cf81f6d4309944e4d1e59ac5b"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:a16418ecf1329f71df119e8a65f3aa68004a3f9383821edcb20f0702934d8087"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:9d9153257a3f70d5f69edf2325357251ed20f772b12e593f3b3377b5f78e7ef8"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:02a51034802cbf38db3f89c66fb5d2ec57e6fe7ef2f4a44d070a593c3688667b"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:2e396d70bc4ef5325b72b593a72c8979999aa52fb8bcf03f701c1b03e1166918"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:11b53acf2411c3b09e6af37e4b9005cba376c872503c8f28218c7243582df45d"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-win32.whl", hash = "sha256:0bf2dae5291758b6f84cf923bfaa285632816007db0330002fa1de38bfcb7154"}, - {file = "charset_normalizer-3.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:2c03cc56021a4bd59be889c2b9257dae13bf55041a3372d3295416f86b295fb5"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:024e606be3ed92216e2b6952ed859d86b4cfa52cd5bc5f050e7dc28f9b43ec42"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4b0d02d7102dd0f997580b51edc4cebcf2ab6397a7edf89f1c73b586c614272c"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:358a7c4cb8ba9b46c453b1dd8d9e431452d5249072e4f56cfda3149f6ab1405e"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:81d6741ab457d14fdedc215516665050f3822d3e56508921cc7239f8c8e66a58"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8b8af03d2e37866d023ad0ddea594edefc31e827fee64f8de5611a1dbc373174"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9cf4e8ad252f7c38dd1f676b46514f92dc0ebeb0db5552f5f403509705e24753"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e696f0dd336161fca9adbb846875d40752e6eba585843c768935ba5c9960722b"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c22d3fe05ce11d3671297dc8973267daa0f938b93ec716e12e0f6dee81591dc1"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:109487860ef6a328f3eec66f2bf78b0b72400280d8f8ea05f69c51644ba6521a"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:37f8febc8ec50c14f3ec9637505f28e58d4f66752207ea177c1d67df25da5aed"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:f97e83fa6c25693c7a35de154681fcc257c1c41b38beb0304b9c4d2d9e164479"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a152f5f33d64a6be73f1d30c9cc82dfc73cec6477ec268e7c6e4c7d23c2d2291"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:39049da0ffb96c8cbb65cbf5c5f3ca3168990adf3551bd1dee10c48fce8ae820"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-win32.whl", hash = "sha256:4457ea6774b5611f4bed5eaa5df55f70abde42364d498c5134b7ef4c6958e20e"}, - {file = "charset_normalizer-3.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:e62164b50f84e20601c1ff8eb55620d2ad25fb81b59e3cd776a1902527a788af"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8eade758719add78ec36dc13201483f8e9b5d940329285edcd5f70c0a9edbd7f"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8499ca8f4502af841f68135133d8258f7b32a53a1d594aa98cc52013fff55678"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3fc1c4a2ffd64890aebdb3f97e1278b0cc72579a08ca4de8cd2c04799a3a22be"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:00d3ffdaafe92a5dc603cb9bd5111aaa36dfa187c8285c543be562e61b755f6b"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c2ac1b08635a8cd4e0cbeaf6f5e922085908d48eb05d44c5ae9eabab148512ca"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f6f45710b4459401609ebebdbcfb34515da4fc2aa886f95107f556ac69a9147e"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ae1de54a77dc0d6d5fcf623290af4266412a7c4be0b1ff7444394f03f5c54e3"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3b590df687e3c5ee0deef9fc8c547d81986d9a1b56073d82de008744452d6541"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ab5de034a886f616a5668aa5d098af2b5385ed70142090e2a31bcbd0af0fdb3d"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9cb3032517f1627cc012dbc80a8ec976ae76d93ea2b5feaa9d2a5b8882597579"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:608862a7bf6957f2333fc54ab4399e405baad0163dc9f8d99cb236816db169d4"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:0f438ae3532723fb6ead77e7c604be7c8374094ef4ee2c5e03a3a17f1fca256c"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:356541bf4381fa35856dafa6a965916e54bed415ad8a24ee6de6e37deccf2786"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-win32.whl", hash = "sha256:39cf9ed17fe3b1bc81f33c9ceb6ce67683ee7526e65fde1447c772afc54a1bb8"}, - {file = "charset_normalizer-3.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:0a11e971ed097d24c534c037d298ad32c6ce81a45736d31e0ff0ad37ab437d59"}, - {file = "charset_normalizer-3.0.1-py3-none-any.whl", hash = "sha256:7e189e2e1d3ed2f4aebabd2d5b0f931e883676e51c7624826e0a4e5fe8a0bf24"}, + {file = "charset-normalizer-3.1.0.tar.gz", hash = "sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d7fc3fca01da18fbabe4625d64bb612b533533ed10045a2ac3dd194bfa656b60"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20064ead0717cf9a73a6d1e779b23d149b53daf971169289ed2ed43a71e8d3b0"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1435ae15108b1cb6fffbcea2af3d468683b7afed0169ad718451f8db5d1aff6f"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c84132a54c750fda57729d1e2599bb598f5fa0344085dbde5003ba429a4798c0"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75f2568b4189dda1c567339b48cba4ac7384accb9c2a7ed655cd86b04055c795"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11d3bcb7be35e7b1bba2c23beedac81ee893ac9871d0ba79effc7fc01167db6c"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:891cf9b48776b5c61c700b55a598621fdb7b1e301a550365571e9624f270c203"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:5f008525e02908b20e04707a4f704cd286d94718f48bb33edddc7d7b584dddc1"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:b06f0d3bf045158d2fb8837c5785fe9ff9b8c93358be64461a1089f5da983137"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:49919f8400b5e49e961f320c735388ee686a62327e773fa5b3ce6721f7e785ce"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:22908891a380d50738e1f978667536f6c6b526a2064156203d418f4856d6e86a"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-win32.whl", hash = "sha256:12d1a39aa6b8c6f6248bb54550efcc1c38ce0d8096a146638fd4738e42284448"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:65ed923f84a6844de5fd29726b888e58c62820e0769b76565480e1fdc3d062f8"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9a3267620866c9d17b959a84dd0bd2d45719b817245e49371ead79ed4f710d19"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6734e606355834f13445b6adc38b53c0fd45f1a56a9ba06c2058f86893ae8017"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aaf53a6cebad0eae578f062c7d462155eada9c172bd8c4d250b8c1d8eb7f916a"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3dc5b6a8ecfdc5748a7e429782598e4f17ef378e3e272eeb1340ea57c9109f41"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e1b25e3ad6c909f398df8921780d6a3d120d8c09466720226fc621605b6f92b1"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ca564606d2caafb0abe6d1b5311c2649e8071eb241b2d64e75a0d0065107e62"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b82fab78e0b1329e183a65260581de4375f619167478dddab510c6c6fb04d9b6"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bd7163182133c0c7701b25e604cf1611c0d87712e56e88e7ee5d72deab3e76b5"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:11d117e6c63e8f495412d37e7dc2e2fff09c34b2d09dbe2bee3c6229577818be"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:cf6511efa4801b9b38dc5546d7547d5b5c6ef4b081c60b23e4d941d0eba9cbeb"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:abc1185d79f47c0a7aaf7e2412a0eb2c03b724581139193d2d82b3ad8cbb00ac"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cb7b2ab0188829593b9de646545175547a70d9a6e2b63bf2cd87a0a391599324"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-win32.whl", hash = "sha256:c36bcbc0d5174a80d6cccf43a0ecaca44e81d25be4b7f90f0ed7bcfbb5a00909"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:cca4def576f47a09a943666b8f829606bcb17e2bc2d5911a46c8f8da45f56755"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0c95f12b74681e9ae127728f7e5409cbbef9cd914d5896ef238cc779b8152373"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac0aa6cd53ab9a31d397f8303f92c42f534693528fafbdb997c82bae6e477ad9"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c3af8e0f07399d3176b179f2e2634c3ce9c1301379a6b8c9c9aeecd481da494f"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a5fc78f9e3f501a1614a98f7c54d3969f3ad9bba8ba3d9b438c3bc5d047dd28"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:628c985afb2c7d27a4800bfb609e03985aaecb42f955049957814e0491d4006d"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:74db0052d985cf37fa111828d0dd230776ac99c740e1a758ad99094be4f1803d"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1e8fcdd8f672a1c4fc8d0bd3a2b576b152d2a349782d1eb0f6b8e52e9954731d"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:dd5653e67b149503c68c4018bf07e42eeed6b4e956b24c00ccdf93ac79cdff84"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d2686f91611f9e17f4548dbf050e75b079bbc2a82be565832bc8ea9047b61c8c"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-win32.whl", hash = "sha256:4155b51ae05ed47199dc5b2a4e62abccb274cee6b01da5b895099b61b1982974"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:322102cdf1ab682ecc7d9b1c5eed4ec59657a65e1c146a0da342b78f4112db23"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e633940f28c1e913615fd624fcdd72fdba807bf53ea6925d6a588e84e1151531"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3a06f32c9634a8705f4ca9946d667609f52cf130d5548881401f1eb2c39b1e2c"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7381c66e0561c5757ffe616af869b916c8b4e42b367ab29fedc98481d1e74e14"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3573d376454d956553c356df45bb824262c397c6e26ce43e8203c4c540ee0acb"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e89df2958e5159b811af9ff0f92614dabf4ff617c03a4c1c6ff53bf1c399e0e1"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:78cacd03e79d009d95635e7d6ff12c21eb89b894c354bd2b2ed0b4763373693b"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de5695a6f1d8340b12a5d6d4484290ee74d61e467c39ff03b39e30df62cf83a0"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c60b9c202d00052183c9be85e5eaf18a4ada0a47d188a83c8f5c5b23252f649"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f645caaf0008bacf349875a974220f1f1da349c5dbe7c4ec93048cdc785a3326"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ea9f9c6034ea2d93d9147818f17c2a0860d41b71c38b9ce4d55f21b6f9165a11"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:80d1543d58bd3d6c271b66abf454d437a438dff01c3e62fdbcd68f2a11310d4b"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:73dc03a6a7e30b7edc5b01b601e53e7fc924b04e1835e8e407c12c037e81adbd"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6f5c2e7bc8a4bf7c426599765b1bd33217ec84023033672c1e9a8b35eaeaaaf8"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-win32.whl", hash = "sha256:12a2b561af122e3d94cdb97fe6fb2bb2b82cef0cdca131646fdb940a1eda04f0"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:3160a0fd9754aab7d47f95a6b63ab355388d890163eb03b2d2b87ab0a30cfa59"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:38e812a197bf8e71a59fe55b757a84c1f946d0ac114acafaafaf21667a7e169e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6baf0baf0d5d265fa7944feb9f7451cc316bfe30e8df1a61b1bb08577c554f31"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8f25e17ab3039b05f762b0a55ae0b3632b2e073d9c8fc88e89aca31a6198e88f"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3747443b6a904001473370d7810aa19c3a180ccd52a7157aacc264a5ac79265e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b116502087ce8a6b7a5f1814568ccbd0e9f6cfd99948aa59b0e241dc57cf739f"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d16fd5252f883eb074ca55cb622bc0bee49b979ae4e8639fff6ca3ff44f9f854"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21fa558996782fc226b529fdd2ed7866c2c6ec91cee82735c98a197fae39f706"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f6c7a8a57e9405cad7485f4c9d3172ae486cfef1344b5ddd8e5239582d7355e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ac3775e3311661d4adace3697a52ac0bab17edd166087d493b52d4f4f553f9f0"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:10c93628d7497c81686e8e5e557aafa78f230cd9e77dd0c40032ef90c18f2230"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:6f4f4668e1831850ebcc2fd0b1cd11721947b6dc7c00bf1c6bd3c929ae14f2c7"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:0be65ccf618c1e7ac9b849c315cc2e8a8751d9cfdaa43027d4f6624bd587ab7e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:53d0a3fa5f8af98a1e261de6a3943ca631c526635eb5817a87a59d9a57ebf48f"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-win32.whl", hash = "sha256:a04f86f41a8916fe45ac5024ec477f41f886b3c435da2d4e3d2709b22ab02af1"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:830d2948a5ec37c386d3170c483063798d7879037492540f10a475e3fd6f244b"}, + {file = "charset_normalizer-3.1.0-py3-none-any.whl", hash = "sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d"}, ] [[package]] @@ -553,14 +540,14 @@ test = ["objgraph", "psutil"] [[package]] name = "identify" -version = "2.5.18" +version = "2.5.19" description = "File identification library for Python" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "identify-2.5.18-py2.py3-none-any.whl", hash = "sha256:93aac7ecf2f6abf879b8f29a8002d3c6de7086b8c28d88e1ad15045a15ab63f9"}, - {file = "identify-2.5.18.tar.gz", hash = "sha256:89e144fa560cc4cffb6ef2ab5e9fb18ed9f9b3cb054384bab4b95c12f6c309fe"}, + {file = "identify-2.5.19-py2.py3-none-any.whl", hash = "sha256:3ee3533e7f6f5023157fbebbd5687bb4b698ce6f305259e0d24b2d7d9efb72bc"}, + {file = "identify-2.5.19.tar.gz", hash = "sha256:4102ecd051f6884449e7359e55b38ba6cd7aafb6ef27b8e2b38495a5723ea106"}, ] [package.extras] @@ -620,25 +607,6 @@ docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker perf = ["ipython"] testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] -[[package]] -name = "importlib-resources" -version = "5.12.0" -description = "Read resources from Python packages" -category = "dev" -optional = false -python-versions = ">=3.7" -files = [ - {file = "importlib_resources-5.12.0-py3-none-any.whl", hash = "sha256:7b1deeebbf351c7578e09bf2f63fa2ce8b5ffec296e0d349139d43cca061a81a"}, - {file = "importlib_resources-5.12.0.tar.gz", hash = "sha256:4be82589bf5c1d7999aedf2a45159d10cb3ca4f19b2271f8792bc8e6da7b22f6"}, -] - -[package.dependencies] -zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] - [[package]] name = "iniconfig" version = "2.0.0" @@ -665,8 +633,6 @@ files = [ [package.dependencies] attrs = ">=17.4.0" -importlib-resources = {version = ">=1.4.0", markers = "python_version < \"3.9\""} -pkgutil-resolve-name = {version = ">=1.3.10", markers = "python_version < \"3.9\""} pyrsistent = ">=0.14.0,<0.17.0 || >0.17.0,<0.17.1 || >0.17.1,<0.17.2 || >0.17.2" [package.extras] @@ -939,28 +905,16 @@ files = [ {file = "pathspec-0.11.0.tar.gz", hash = "sha256:64d338d4e0914e91c1792321e6907b5a593f1ab1851de7fc269557a21b30ebbc"}, ] -[[package]] -name = "pkgutil-resolve-name" -version = "1.3.10" -description = "Resolve a name to an object." -category = "dev" -optional = false -python-versions = ">=3.6" -files = [ - {file = "pkgutil_resolve_name-1.3.10-py3-none-any.whl", hash = "sha256:ca27cc078d25c5ad71a9de0a7a330146c4e014c2462d9af19c6b828280649c5e"}, - {file = "pkgutil_resolve_name-1.3.10.tar.gz", hash = "sha256:357d6c9e6a755653cfd78893817c0853af365dd51ec97f3d358a819373bbd174"}, -] - [[package]] name = "platformdirs" -version = "3.0.0" +version = "3.1.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "platformdirs-3.0.0-py3-none-any.whl", hash = "sha256:b1d5eb14f221506f50d6604a561f4c5786d9e80355219694a1b244bcd96f4567"}, - {file = "platformdirs-3.0.0.tar.gz", hash = "sha256:8a1228abb1ef82d788f74139988b137e78692984ec7b08eaa6c65f1723af28f9"}, + {file = "platformdirs-3.1.0-py3-none-any.whl", hash = "sha256:13b08a53ed71021350c9e300d4ea8668438fb0046ab3937ac9a29913a1a1350a"}, + {file = "platformdirs-3.1.0.tar.gz", hash = "sha256:accc3665857288317f32c7bebb5a8e482ba717b474f3fc1d18ca7f9214be0cef"}, ] [package.extras] @@ -1333,20 +1287,19 @@ files = [ [[package]] name = "rich" -version = "13.3.1" +version = "13.3.2" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" category = "dev" optional = false python-versions = ">=3.7.0" files = [ - {file = "rich-13.3.1-py3-none-any.whl", hash = "sha256:8aa57747f3fc3e977684f0176a88e789be314a99f99b43b75d1e9cb5dc6db9e9"}, - {file = "rich-13.3.1.tar.gz", hash = "sha256:125d96d20c92b946b983d0d392b84ff945461e5a06d3867e9f9e575f8697b67f"}, + {file = "rich-13.3.2-py3-none-any.whl", hash = "sha256:a104f37270bf677148d8acb07d33be1569eeee87e2d1beb286a4e9113caf6f2f"}, + {file = "rich-13.3.2.tar.gz", hash = "sha256:91954fe80cfb7985727a467ca98a7618e5dd15178cc2da10f553b36a93859001"}, ] [package.dependencies] -markdown-it-py = ">=2.1.0,<3.0.0" -pygments = ">=2.14.0,<3.0.0" -typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9\""} +markdown-it-py = ">=2.2.0,<3.0.0" +pygments = ">=2.13.0,<3.0.0" [package.extras] jupyter = ["ipywidgets (>=7.5.1,<9)"] @@ -1379,14 +1332,14 @@ files = [ [[package]] name = "setuptools" -version = "67.4.0" +version = "67.6.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "setuptools-67.4.0-py3-none-any.whl", hash = "sha256:f106dee1b506dee5102cc3f3e9e68137bbad6d47b616be7991714b0c62204251"}, - {file = "setuptools-67.4.0.tar.gz", hash = "sha256:e5fd0a713141a4a105412233c63dc4e17ba0090c8e8334594ac790ec97792330"}, + {file = "setuptools-67.6.0-py3-none-any.whl", hash = "sha256:b78aaa36f6b90a074c1fa651168723acbf45d14cb1196b6f02c0fd07f17623b2"}, + {file = "setuptools-67.6.0.tar.gz", hash = "sha256:2ee892cd5f29f3373097f5a814697e397cf3ce313616df0af11231e2ad118077"}, ] [package.extras] @@ -1474,7 +1427,7 @@ greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and (platfo [package.extras] aiomysql = ["aiomysql", "greenlet (!=0.4.17)"] -aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing-extensions (!=3.10.0.1)"] +aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing_extensions (!=3.10.0.1)"] asyncio = ["greenlet (!=0.4.17)"] asyncmy = ["asyncmy (>=0.2.3,!=0.2.4)", "greenlet (!=0.4.17)"] mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2)"] @@ -1484,14 +1437,14 @@ mssql-pyodbc = ["pyodbc"] mypy = ["mypy (>=0.910)", "sqlalchemy2-stubs"] mysql = ["mysqlclient (>=1.4.0)", "mysqlclient (>=1.4.0,<2)"] mysql-connector = ["mysql-connector-python"] -oracle = ["cx-oracle (>=7)", "cx-oracle (>=7,<8)"] +oracle = ["cx_oracle (>=7)", "cx_oracle (>=7,<8)"] postgresql = ["psycopg2 (>=2.7)"] postgresql-asyncpg = ["asyncpg", "greenlet (!=0.4.17)"] postgresql-pg8000 = ["pg8000 (>=1.16.6,!=1.29.0)"] postgresql-psycopg2binary = ["psycopg2-binary"] postgresql-psycopg2cffi = ["psycopg2cffi"] pymysql = ["pymysql", "pymysql (<1)"] -sqlcipher = ["sqlcipher3-binary"] +sqlcipher = ["sqlcipher3_binary"] [[package]] name = "tabulate" @@ -1762,7 +1715,7 @@ files = [ [[package]] name = "yardstick" -version = "0.3.4.post12.dev0+6533a41" +version = "0.3.4.post23.dev0+0ed40aa" description = "Tool for comparing the results from vulnerability scanners" category = "dev" optional = false @@ -1810,5 +1763,5 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" -python-versions = ">=3.8.1,<4.0" -content-hash = "37098f9507fad3438112e3683e503a5c5a55c2ec3989e981b73ee0bb5c2b38e9" +python-versions = "^3.9" +content-hash = "6ce4b7d1a055847b41258f9b619628b3e36d0d5f2a65a7632f7a21c44f462b89" diff --git a/pyproject.toml b/pyproject.toml index 4752cc28..39786541 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,8 @@ exclude = [ ] [tool.poetry.dependencies] -python = ">=3.8.1,<4.0" +python = "^3.9" + click = "^8.1.3" requests = "^2.28.1" colorlog = "^6.7.0" @@ -57,7 +58,11 @@ requires = ["poetry-core>=1.3.0", "poetry-dynamic-versioning"] # note: this is a thin wrapper around "poetry.core.masonry.api" build-backend = "poetry_dynamic_versioning.backend" +[tool.pytest.ini_options] +cache_dir = ".cache/pytest" + [tool.mypy] +cache_dir = ".cache/mypy" follow_imports = "silent" strict_optional = true warn_redundant_casts = true @@ -134,7 +139,7 @@ style = "semver" dirty = true [tool.ruff] - +cache-dir = ".cache/ruff" # allow for a wide-birth relative to what black will correct to line-length = 150 select = [ diff --git a/src/vunnel/provider.py b/src/vunnel/provider.py index 9c83c50d..d1d83a1b 100644 --- a/src/vunnel/provider.py +++ b/src/vunnel/provider.py @@ -39,10 +39,15 @@ def __repr__(self) -> str: @dataclass class OnErrorConfig: + # the action to take when an error occurs action: OnErrorAction = OnErrorAction.FAIL + # the number of times to retry an action that fails retry_count: int = 3 + # the number of seconds to wait between retries retry_delay: int = 5 + # what to do with the input directory when an error occurs input: InputStatePolicy = InputStatePolicy.KEEP # noqa: A003 + # what to do with the result directory when an error occurs results: ResultStatePolicy = ResultStatePolicy.KEEP def __post_init__(self) -> None: @@ -57,9 +62,13 @@ def __post_init__(self) -> None: @dataclass class RuntimeConfig: + # what to do when an error occurs while the provider is running on_error: OnErrorConfig = field(default_factory=OnErrorConfig) + # what to do with existing data in the input directory before running existing_input: InputStatePolicy = InputStatePolicy.KEEP + # what to do with existing data in the result directory before running existing_results: ResultStatePolicy = ResultStatePolicy.KEEP + # the format the results should be written in result_store: result.StoreStrategy = result.StoreStrategy.FLAT_FILE def __post_init__(self) -> None: