fix: some invalid CVE data was loaded into the cache

deploy/onguard-redis.yaml

@@ -28,9 +28,14 @@ spec:
         volumeMounts:
           - name: logs
             mountPath: /redisinsight/logs
+          - name: data
+            mountPath: /data
       volumes:
         - name: logs
           emptyDir: {}
+        - name: data
+          persistentVolumeClaim:
+            claimName: redis-data
 ---
 apiVersion: v1
 kind: Service
@@ -44,9 +49,21 @@ spec:
     port: 6379
     protocol: TCP
     targetPort: 6379
-  - name: management
+  - name: http
     port: 8001
     protocol: TCP
     targetPort: 8001
   selector:
-    app: redis
+    app: redis
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: redis-data
+spec:
+  storageClassName: gp2
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 4Gi

deploy/onguard.yaml

@@ -89,27 +89,39 @@ data:
 
     if [ ${status_code} == '404' ]; then
         echo Load data is empty
-        curl -XPOST ${endpoint}
+        curl -s -XPOST ${endpoint}
     elif [ ${status_code} == '200' ]; then
         load=$(curl -s ${endpoint})
         echo ${load}
         status=$(echo "$load" | sed -E 's/.*"status":"?([^,"]*)"?.*/\1/')
         if [[ $status == "COMPLETED" ]]; then
             completed=$(echo "$load" | sed -E 's/.*"completed":"?([^,"]*)"?.*/\1/')
             echo "Completed load on ${completed}. Sync with most recent data"
-            curl -XPOST ${endpoint}?since=${completed}
+            curl -s -XPOST ${endpoint}?since=${completed}
         elif [[ $status == "COMPLETED_WITH_ERRORS" ]]; then
             echo "Previous load completed with errors. Resume"
-            curl -XPOST ${endpoint}
+            curl -s -XPOST ${endpoint}
         elif [[ $status == "PROCESSING" ]]; then
             echo "There is an ongoing load. Skip"
         else
             echo "Unknown load status ${load}. Force reload"
-            curl -XPOST ${endpoint}?reload=true
+            curl -s -XPOST ${endpoint}?reload=true
         fi
     else
         echo "Unexpected status code ${status_code}. Skip load and sync."
     fi
+  force_reload.sh: |
+    #!/bin/bash
+    set -euo pipefail
+    endpoint=http://onguard:9000/load
+    force_reload_date=2024-02-20T00:00:00
+
+    for i in {1..5}; do
+      echo retry $i
+      curl -s -XPOST ${endpoint}?reload_before=${force_reload_date} && exit 0
+      sleep 1
+    done
+    exit 1
 ---
 apiVersion: batch/v1
 kind: CronJob
@@ -138,3 +150,28 @@ spec:
                 defaultMode: 0777
           restartPolicy: OnFailure
 
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: reload-nvd-db
+spec:
+  template:
+    spec:
+      ttlSecondsAfterFinished: 3600
+      containers:
+        - name: sync
+          image: registry.access.redhat.com/ubi9/ubi-minimal:9.3
+          imagePullPolicy: IfNotPresent
+          command:
+            - /bin/sh
+            - /scripts/force_reload.sh
+          volumeMounts:
+            - name: script-cm
+              mountPath: /scripts
+      volumes:
+        - name: script-cm
+          configMap:
+            name: sync-nvd-script
+            defaultMode: 0777
+      restartPolicy: OnFailure

deploy/openshift/template.yaml

@@ -131,27 +131,39 @@ objects:
 
         if [ ${status_code} == '404' ]; then
             echo Load data is empty
-            curl -XPOST ${endpoint}
+            curl -s -XPOST ${endpoint}
         elif [ ${status_code} == '200' ]; then
             load=$(curl -s ${endpoint})
             echo ${load}
             status=$(echo "$load" | sed -E 's/.*"status":"?([^,"]*)"?.*/\1/')
             if [[ $status == "COMPLETED" ]]; then
                 completed=$(echo "$load" | sed -E 's/.*"completed":"?([^,"]*)"?.*/\1/')
                 echo "Completed load on ${completed}. Sync with most recent data"
-                curl -XPOST ${endpoint}?since=${completed}
+                curl -s -XPOST ${endpoint}?since=${completed}
             elif [[ $status == "COMPLETED_WITH_ERRORS" ]]; then
                 echo "Previous load completed with errors. Resume"
-                curl -XPOST ${endpoint}
+                curl -s -XPOST ${endpoint}
             elif [[ $status == "PROCESSING" ]]; then
                 echo "There is an ongoing load. Skip"
             else
                 echo "Unknown load status ${load}. Force reload."
-                curl -XPOST ${endpoint}?reload=true
+                curl -s -XPOST ${endpoint}?reload=true
             fi
         else
             echo "Unexpected status code ${status_code}. Skip load and sync."
         fi
+      force_reload.sh: |
+        #!/bin/bash
+        set -euo pipefail
+        endpoint=http://onguard:9000/load
+        force_reload_date=2024-02-20T00:00:00
+
+        for i in {1..5}; do
+          echo retry $i
+          curl -s -XPOST ${endpoint}?reload_before=${force_reload_date} && exit 0
+          sleep 1
+        done
+        exit 1
   - apiVersion: batch/v1
     kind: CronJob
     metadata:
@@ -178,6 +190,29 @@ objects:
                     name: sync-nvd-script
                     defaultMode: 0777
               restartPolicy: OnFailure
+  - apiVersion: batch/v1
+    kind: Job
+    metadata:
+      name: force-reload-nvd-db
+    spec:
+      template:
+        spec:
+          containers:
+            - name: sync
+              image: quay.io/app-sre/ubi9-ubi-minimal:9.3
+              imagePullPolicy: IfNotPresent
+              command:
+                - /bin/sh
+                - /scripts/force_reload.sh
+              volumeMounts:
+                - name: script-cm
+                  mountPath: /scripts
+          volumes:
+            - name: script-cm
+              configMap:
+                name: sync-nvd-script
+                defaultMode: 0777
+          restartPolicy: OnFailure
 parameters:
   - name: APP_NAME
     displayName: Application name

src/main/java/com/redhat/ecosystemappeng/onguard/repository/VulnerabilityRepository.java

@@ -35,4 +35,6 @@ public interface VulnerabilityRepository {
     List<VulnerabilityAlias> listByAliases(List<String> aliases);
 
     void setAliases(List<String> aliases, String cveId);
+
+    void removeAll();
 }

src/main/java/com/redhat/ecosystemappeng/onguard/repository/redis/VulnerabilityRedisRepository.java

@@ -80,7 +80,9 @@ public void save(Vulnerability vulnerability) {
     }
 
     public void setAliases(List<String> aliases, String cveId) {
+      if(aliases != null && !aliases.isEmpty() && cveId != null) {
         aliasCommands.mset(aliases.stream().collect(Collectors.toMap(v -> Alias.getKey(v), v -> new Alias(v, cveId))));
+      }
     }
 
     @Override
@@ -142,4 +144,9 @@ public List<VulnerabilityAlias> listByAliases(List<String> aliasIds) {
         return results;
     }
 
+    @Override
+    public void removeAll() {
+      aliasCommands.getDataSource().flushall();
+    }
+
 }

src/main/java/com/redhat/ecosystemappeng/onguard/rest/LoadRoute.java

@@ -17,9 +17,9 @@
  */
 package com.redhat.ecosystemappeng.onguard.rest;
 
-import java.text.ParseException;
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeParseException;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -49,22 +49,27 @@ public class LoadRoute {
     public void registerManagementRoutes(@Observes ManagementInterface mi) {
         mi.router().post("/load").handler(ctx -> {
             final LocalDateTime since;
+            final LocalDateTime reloadBefore;
             try {
                 since = parseDateParam(ctx.request().getParam("since"));
-            } catch (ParseException e) {
+                reloadBefore = parseDateParam(ctx.request().getParam("reload_before"));
+            } catch (DateTimeParseException e) {
                 ctx.response().setStatusCode(400).setStatusMessage(e.getMessage()).end();
                 return;
             }
             Boolean reload = Boolean.TRUE.toString().equalsIgnoreCase(ctx.request().getParam("reload"));
             Uni.createFrom()
                     .voidItem()
                     .invoke(() -> {
-                        if(reload) {
+                        if(reloadBefore != null) {
+                          var bulk = loadService.get();
+                          if(bulk != null && bulk.completed()!= null && reloadBefore.isBefore(bulk.completed())) {
+                            loadService.restart();
+                          }
+                        } else if(reload) {
                             loadService.restart();
-                        } else {
-                            loadService.loadFromNvdApi(since);
                         }
-
+                        loadService.loadFromNvdApi(since);
                     })
                     .runSubscriptionOn(Infrastructure.getDefaultExecutor())
                     .subscribeAsCompletionStage()
@@ -107,7 +112,7 @@ public void registerManagementRoutes(@Observes ManagementInterface mi) {
         });
     }
 
-    private LocalDateTime parseDateParam(String param) throws ParseException {
+    private LocalDateTime parseDateParam(String param) throws DateTimeParseException {
         if(param != null) {
            return LocalDateTime.parse(param, DateTimeFormatter.ISO_LOCAL_DATE_TIME);
         }

src/main/java/com/redhat/ecosystemappeng/onguard/service/LoadService.java

@@ -70,16 +70,13 @@ public void loadFromNvdApi(LocalDateTime since) {
                 Multi.createFrom().items(vulnerabilities.stream())
                         .runSubscriptionOn(Infrastructure.getDefaultExecutor()).subscribe()
                         .with(vulnerabilityService::ingestNvdVulnerability);
-                vulnerabilities.stream().parallel().forEach(vulnerabilityService::ingestNvdVulnerability);
-
                 synchronized (this) {
                     bulk = bulkRepository.get();
                     var builder = Bulk.builder(bulk).index(bulk.index() + loaded).pageSize(pageSize);
                     if (loaded < pageSize) {
                         var completed = LocalDateTime.now(ZoneId.systemDefault());
                         builder.completed(completed)
                                 .status(Status.COMPLETED);
-
                     }
                     bulkRepository.set(builder.build());
                 }
@@ -114,8 +111,7 @@ public synchronized Bulk cancel() {
     }
 
     public void restart() {
-        bulkRepository.remove();
-        loadFromNvdApi(null);
+        vulnerabilityService.cleanUp();
     }
 
 }

src/main/java/com/redhat/ecosystemappeng/onguard/service/VulnerabilityService.java

@@ -32,4 +32,6 @@ public interface VulnerabilityService {
 
     void ingestNvdVulnerability(Vulnerability vuln);
 
+    void cleanUp();
+
 }

src/main/java/com/redhat/ecosystemappeng/onguard/service/VulnerabilityServiceImpl.java

@@ -97,7 +97,7 @@ public Map<String, List<Vulnerability>> findByPurls(List<String> purls, boolean
 
   @Override
   public void ingestNvdVulnerability(Vulnerability vuln) {
-    if (vuln == null) {
+    if (vuln == null || !OsvVulnerability.CVE_PATTERN.matcher(vuln.cveId()).matches()) {
       return;
     }
     LOGGER.debug("Ingest NVD Vulnerability {}", vuln.cveId());
@@ -161,4 +161,9 @@ private Vulnerability getOsvVulnerabilityData(String alias) {
         .affected(osvVuln.affected()).build();
   }
 
+  @Override
+  public void cleanUp() {
+    repository.removeAll();
+  }
+
 }

src/main/java/com/redhat/ecosystemappeng/onguard/service/nvd/NvdApi.java

@@ -39,7 +39,7 @@
 public interface NvdApi {
 
     static final long NVD_API_WINDOW_SECS = 30;
-    static final String SIMPLE_ISO_8601 = "yyyy-MM-dd";
+
     @GET
     @ClientHeaderParam(name = "apiKey", value = "${api.nvd.apikey}")
     @Produces(MediaType.APPLICATION_JSON)