Parser fixes: CycloneDX dict input, Trivy CVE dedup, and test coverage

.github/workflows/cicd-actions.yml

@@ -26,10 +26,13 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Create .env file
+        env:
+          POSTGRES_PASSWORD: ${{ secrets.TEST_DB_PASSWORD }}
+          SALT: ${{ secrets.TEST_SALT }}
         run: |
           echo "NVD_API_KEY=${{ secrets.TEST_NVD_API_KEY }}" >> .env
           echo 'DJANGO_SECRET_KEY="${{ secrets.TEST_DJANGO_SECRET_KEY }}"' >> .env
-          echo 'SALT="${{ secrets.TEST_SALT }}"' >> .env
+          echo "SALT=${SALT:-local-dev-salt}" >> .env
           echo "ADMIN_USERNAME=admin@acme.de" >> .env
           echo "ADMIN_PASSWORD=secure!" >> .env
           echo "USER_USERNAME=user@acme.de" >> .env
@@ -41,7 +44,7 @@ jobs:
           echo "POSTGRES_USER=securecheckplus" >> .env
           echo "POSTGRES_DB=securecheckplus" >> .env
           echo "POSTGRES_PORT=5432" >> .env
-          echo 'POSTGRES_PASSWORD="${{ secrets.TEST_DB_PASSWORD }}"' >> .env
+          echo "POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-scp_test_pass}" >> .env
           echo "EMAIL_HOST=localhost" >> .env
           echo "EMAIL_PORT=25" >> .env
           echo 'LDAP_ORGANISATION="ACME"' >> .env
@@ -77,6 +80,7 @@ jobs:
           path: backend
 
       - name: Docker Login
+        if: env.DOCKER_USER != '' && env.DOCKER_KEY != ''
         run: echo "$DOCKER_KEY" | docker login -u "$DOCKER_USER" --password-stdin
 
       - name: Build Docker Compose
@@ -182,6 +186,7 @@ jobs:
           path: backend/assets
 
       - name: Docker Login
+        if: env.DOCKER_USER != '' && env.DOCKER_KEY != ''
         run: echo "$DOCKER_KEY" | docker login -u "$DOCKER_USER" --password-stdin
 
       - name: Extract metadata (tags, labels) for Docker
@@ -221,6 +226,7 @@ jobs:
           path: backend
 
       - name: Docker Login
+        if: env.DOCKER_USER != '' && env.DOCKER_KEY != ''
         run: echo "$DOCKER_KEY" | docker login -u "$DOCKER_USER" --password-stdin
 
       - name: Extract metadata (tags, labels) for Docker

backend/analyzer/parser/cyclonedx_parser.py

@@ -24,10 +24,13 @@ def get_package_manager_name(bom_ref: str):
     return "NA"
 
 
-def parse_json(json_data: str) -> dict[str, ParseResult]:
+def parse_json(json_data: str or dict) -> dict[str, ParseResult]:
   try:
     # deserialize
-    bom = cast(Bom, Bom.from_json(data=json.loads(json_data)))
+    if isinstance(json_data, dict):
+        bom = cast(Bom, Bom.from_json(data=json_data))
+    else:
+        bom = cast(Bom, Bom.from_json(data=json.loads(json_data)))
     data: dict[str, ParseResult] = {}
 
     # key is affected bom-ref and values are vulnerability ids

backend/analyzer/parser/trivy_parser.py

@@ -56,10 +56,12 @@ def parse_json(json_data: str) -> dict[str, ParseResult]:
                         dependency_name = vul.get("PkgName")
                         vul_id = vul.get("VulnerabilityID")
 
-                        if dependency_name in data:
-                            data.get(dependency_name).vulnerabilities.append(vul_id)
+                        key = f"{dependency_name}:{version}"
+                        if key in data:
+                            if vul_id not in data[key].vulnerabilities:
+                                data[key].vulnerabilities.append(vul_id)
                         else:
-                            data[dependency_name] = ParseResult(
+                            data[key] = ParseResult(
                                 dependency_name=dependency_name,
                                 version=version,
                                 package_manager=package_manager,

backend/analyzer/services/cve_fetcher.py

@@ -54,6 +54,7 @@ class CVEFetcher:
         "baseScore": "N/A",
         "baseSeverity": "N/A"
     }
+    MAX_RETRIES = 3
 
     def __init__(self, cve_id: str):
         """
@@ -71,7 +72,8 @@ def fetch_from_nist_gov(self):
         Fetches CVE data from the NIST government server for the given CVE ID.
 
         Sends a GET request to the NIST API using the CVE ID, processes the response,
-        and updates the `data` attribute with relevant information.
+        and updates the `data` attribute with relevant information. Retries with
+        exponential backoff on server errors (5xx).
 
         Raises:
             ValueError: If the response structure is invalid or missing expected data.
@@ -80,11 +82,28 @@ def fetch_from_nist_gov(self):
             headers = {"apiKey": NVD_API_KEY}
             url = parse.urlunparse(NVD_ADDRESS) + self.cve_id
             logger.info(f"Fetching CVE data from NIST for CVE ID: {self.cve_id} using URL: {url}")
-            response = requests.get(url, headers=headers)
 
-            if response.status_code != 200:
-                logger.warning(
-                    f"Failed to fetch CVE data for CVE ID: {self.cve_id}. HTTP status: {response.status_code}")
+            response = None
+            for attempt in range(self.MAX_RETRIES):
+                response = requests.get(url, headers=headers)
+
+                if response.status_code == 200:
+                    break
+
+                if attempt < self.MAX_RETRIES - 1 and response.status_code >= 500:
+                    delay = 2 ** (attempt + 1)
+                    logger.warning(
+                        f"NVD API returned {response.status_code} for {self.cve_id} "
+                        f"(attempt {attempt + 1}/{self.MAX_RETRIES}), retrying in {delay}s"
+                    )
+                    time.sleep(delay)
+                else:
+                    logger.warning(
+                        f"Failed to fetch CVE data for CVE ID: {self.cve_id}. HTTP status: {response.status_code}"
+                    )
+                    return
+
+            if response is None:
                 return
 
             response_json = response.json()

backend/analyzer/test/conftest.py

@@ -0,0 +1,65 @@
+from unittest.mock import patch
+
+import pytest
+
+MOCK_NVD_RESPONSE = {
+    "vulnerabilities": [
+        {
+            "cve": {
+                "id": "CVE-2021-44228",
+                "published": "2025-01-01T00:00:00Z",
+                "lastModified": "2025-01-01T00:00:00Z",
+                "descriptions": [{"value": "A test vulnerability description."}],
+                "metrics": {
+                    "cvssMetricV31": [
+                        {
+                            "cvssData": {
+                                "baseScore": 7.5,
+                                "baseSeverity": "HIGH",
+                                "attackVector": "NETWORK",
+                                "attackComplexity": "LOW",
+                                "privilegesRequired": "NONE",
+                                "userInteraction": "NONE",
+                                "confidentialityImpact": "HIGH",
+                                "integrityImpact": "NONE",
+                                "availabilityImpact": "NONE",
+                                "scope": "UNCHANGED",
+                            }
+                        }
+                    ]
+                },
+                "weaknesses": [{"description": [{"value": "CWE-94"}]}],
+                "references": [{"url": "https://example.com/advisory", "tags": ["Vendor Advisory"]}],
+            }
+        }
+    ]
+}
+
+MOCK_EPSS_RESPONSE = {"data": [{"epss": 0.5}]}
+
+
+def _mock_requests_get(url, **kwargs):
+    class MockResponse:
+        def __init__(self, json_data, status_code):
+            self.json_data = json_data
+            self.status_code = status_code
+
+        def json(self):
+            return self.json_data
+
+    url_str = str(url)
+    if "nvd.nist.gov" in url_str:
+        return MockResponse(MOCK_NVD_RESPONSE, 200)
+    if "api.first.org" in url_str:
+        return MockResponse(MOCK_EPSS_RESPONSE, 200)
+    raise ConnectionError(f"Unexpected request: {url_str}")
+
+
+@pytest.fixture(autouse=True)
+def no_nvd_network(request):
+    if request.node.get_closest_marker("nvd_integration"):
+        yield
+        return
+    with patch("analyzer.services.cve_fetcher.requests.get", side_effect=_mock_requests_get), \
+         patch("analyzer.services.cve_fetcher.time.sleep"):
+        yield