improve speed of ubuntu driver + various fixes

Signed-off-by: Alex Goodman <[email protected]>

.gitignore

@@ -1,5 +1,6 @@
 .vunnel.yaml
-./data/
+/data/
+/backup/
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

.pre-commit-config.yaml

@@ -45,18 +45,18 @@ repos:
     hooks:
       - id: system
         name: black
-        entry: poetry run black .
+        entry: poetry run black src tests
         pass_filenames: false
         language: system
 
   - repo: local
     hooks:
       - id: system
         name: isort
-        entry: poetry run isort .
+        entry: poetry run isort tests src --filter-files
         pass_filenames: false
         language: system
-
+        files: \.py$
   - repo: local
     hooks:
       - id: system

pyproject.toml

@@ -23,8 +23,8 @@ ijson = "^3.1.4"
 xxhash = "^3.1.0"
 cvss = "^2.5"
 python-dateutil = "^2.8.2"
-dacite = "^1.6.0"
 defusedxml = "^0.7.1"
+dataclass-wizard = "^0.22.2"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.2.0"
@@ -57,6 +57,7 @@ force_grid_wrap = 0
 use_parentheses = true
 ensure_newline_before_comments = true
 line_length = 100
+skip_gitignore = true
 
 [tool.pylint.messages_control]
 disable = [
@@ -121,11 +122,6 @@ exclude = '''(?x)(
     | ^tests/.*$     # any tests
 )'''
 
-[[tool.mypy.overrides]]
-# https://github.com/konradhalas/dacite/issues/133
-module = "dacite"
-implicit_reexport = true
-
 [tool.black]
 line-length = 130
 exclude = '''
@@ -141,6 +137,8 @@ exclude = '''
     | buck-out
     | build
     | dist
+    | data
+    | backup
   )/
 )
 '''

schema/vulnerability/os/schema-1.0.0.json

@@ -107,6 +107,31 @@
         "Metadata": {
           "type": "object",
           "properties": {
+            "Issued": {
+              "type": "string"
+            },
+            "RefId": {
+              "type": "string"
+            },
+            "CVE": {
+              "type": "array",
+              "items": [
+                {
+                  "type": "object",
+                  "properties": {
+                    "Name": {
+                      "type": "string"
+                    },
+                    "Link": {
+                      "type": "string"
+                    }
+                  },
+                  "required": [
+                    "Name"
+                  ]
+                }
+              ]
+            },
             "NVD": {
               "type": "object",
               "properties": {

src/vunnel/__main__.py

@@ -0,0 +1,3 @@
+from vunnel.cli import run
+
+run()

src/vunnel/cli/cli.py

@@ -27,7 +27,7 @@ def cli(ctx, verbose: bool, config_path: str) -> None:  # type: ignore
 
     log_format = "%(log_color)s %(asctime)s %(name)s [%(levelname)s] %(message)s"
     if ctx.obj.log.slim:
-        log_format = "%(log_color)s [%(levelname)s] %(message)s"
+        log_format = "%(log_color)s %(message)s"
 
     logging.config.dictConfig(
         {
@@ -129,7 +129,7 @@ def status_provider(cfg: config.Application, provider_names: str) -> None:
     ├── Inputs:  {len(state.input.files)} files
     │            {state.input.timestamp}
     └── Results: {len(state.results.files)} files
-                {state.results.timestamp}
+                 {state.results.timestamp}
 """
             print(tmpl)
         except FileNotFoundError:

src/vunnel/cli/config.py

@@ -2,10 +2,10 @@
 from dataclasses import dataclass, field, fields
 from typing import Any
 
-import dacite
 import yaml
+from dataclass_wizard import fromdict
 
-from vunnel import provider, providers
+from vunnel import providers
 
 
 @dataclass
@@ -53,19 +53,9 @@ def load(path: str = ".vunnel.yaml") -> Application:  # noqa
     try:
         with open(path, encoding="utf-8") as f:
             app_object = yaml.safe_load(f.read())
-            cfg = dacite.from_dict(
+            cfg = fromdict(
                 Application,
                 app_object,
-                config=dacite.Config(
-                    cast=[
-                        provider.OnErrorAction,
-                        provider.InputStatePolicy,
-                        provider.ResultStatePolicy,
-                    ],
-                    # type_hooks={
-                    #
-                    # }
-                ),
             )
             if cfg is None:
                 raise FileNotFoundError("parsed empty config")

src/vunnel/providers/alpine/parser.py

@@ -76,16 +76,16 @@ def _download(self, skip_if_exists=False):
             os.remove(os.path.join(self.secdb_dir_path, "alpine-secdb-master.tar.gz"))
 
         if skip_if_exists and os.path.exists(self.secdb_dir_path):
-            self.logger.warning(
-                "skip_if_exists flag enabled and found source under {}. Skipping download".format(self.secdb_dir_path)
+            self.logger.debug(
+                "'skip_if_exists' flag enabled and found source under {}. Skipping download".format(self.secdb_dir_path)
             )
         else:
             links = []
             try:
                 if not os.path.exists(self.secdb_dir_path):
                     os.makedirs(self.secdb_dir_path, exist_ok=True)
 
-                self.logger.info("Downloading alpine secdb metadata from: {}".format(self.metadata_url))
+                self.logger.info("downloading alpine secdb metadata from: {}".format(self.metadata_url))
                 r = requests.get(self.metadata_url, timeout=self.download_timeout)
                 if r.status_code == 200:
                     try:
@@ -94,21 +94,21 @@ def _download(self, skip_if_exists=False):
                         parser.feed(r.text)
                         links = parser.links
                     except:
-                        self.logger.warning("Unable to html parse secdb landing page content for links")
+                        self.logger.warning("unable to html parse secdb landing page content for links")
 
                     if not links:
-                        self.logger.debug("String parsing secdb landing page content for links")
+                        self.logger.debug("string parsing secdb landing page content for links")
                         links = re.findall(self._link_finder_regex_, r.text)
                 else:
                     r.raise_for_status()
             except Exception:
-                self.logger.exception("Error downloading or parsing alpine secdb metadata")
+                self.logger.exception("error downloading or parsing alpine secdb metadata")
                 raise
 
             if links:
-                self.logger.debug("Found release specific secdb links: {}".format(links))
+                self.logger.debug("found release specific secdb links: {}".format(links))
             else:
-                raise Exception("Unable to find release specific secdb links")
+                raise Exception("unable to find release specific secdb links")
 
             for link in links:
                 if link not in ignore_links:
@@ -130,7 +130,7 @@ def _download(self, skip_if_exists=False):
                             else:
                                 r.raise_for_status()
                     except:
-                        self.logger.exception("Ignoring error processing secdb for {}".format(link))
+                        self.logger.exception("ignoring error processing secdb for {}".format(link))
 
     def _load(self):
         """
@@ -156,7 +156,7 @@ def _load(self):
                         for dbtype in self._db_types:
                             secdb_yaml_path = os.path.join(self.secdb_dir_path, f, "{}.yaml".format(dbtype))
                             if os.path.exists(secdb_yaml_path):
-                                self.logger.debug("Loading secdb data from: {}".format(secdb_yaml_path))
+                                self.logger.debug("loading secdb data from: {}".format(secdb_yaml_path))
                                 with open(secdb_yaml_path, "r") as FH:
                                     yaml_data = yaml.safe_load(FH)
                                     dbtype_data_dict[dbtype] = yaml_data
@@ -180,7 +180,7 @@ def _normalize(self, release, dbtype_data_dict):
         vuln_dict = {}
 
         for dbtype, data in dbtype_data_dict.items():
-            self.logger.debug("Normalizing {}:{}".format(release, dbtype))
+            self.logger.info("processing {}:{}".format(release, dbtype))
 
             if data["packages"]:
                 for el in data["packages"]:
@@ -251,6 +251,5 @@ def get(self, skip_if_exists: bool = False):
         self._download(skip_if_exists)
 
         for release, dbtype_data_dict in self._load():
-            print(release)
             # normalize the loaded data
             yield release, self._normalize(release, dbtype_data_dict)

src/vunnel/providers/amazon/parser.py

@@ -261,7 +261,13 @@ def map_to_vulnerability(version, alas, fixed_in):
     v.NamespaceName = namespace + ":" + version
     v.Description = ""
     v.Severity = severity_map.get(alas.sev, "Unknown")
-    v.Metadata = {"CVE": alas.cves if alas.cves else []}
+    v.Metadata = {
+        "CVE": [],
+    }
+
+    if alas.cves:
+        v.Metadata["CVE"] = [{"Name": cve} for cve in alas.cves]
+
     v.Link = alas.url
     for item in fixed_in:
         f = FixedIn()

src/vunnel/providers/ubuntu/__init__.py

@@ -13,6 +13,7 @@ class Config:
     request_timeout: int = 125
     additional_versions: dict[str, str] = field(default_factory=lambda: {})
     enable_rev_history: bool = True
+    max_workers: int = 5
 
 
 class Provider(provider.Provider):
@@ -28,6 +29,7 @@ def __init__(self, root: str, config: Config):
             logger=self.logger,
             additional_versions=self.config.additional_versions,
             enable_rev_history=self.config.enable_rev_history,
+            max_workers=self.config.max_workers,
         )
 
     @classmethod
@@ -41,7 +43,7 @@ def update(self) -> list[str]:
                 writer.write(
                     identifier=f"{namespace}-{vuln_id}".lower(),
                     schema=self.schema,
-                    payload=record,
+                    payload={"Vulnerability": record},
                 )
 
         return self.parser.urls