From bce63de45e96ddf4edf8ffa8f267fdbc84ff910c Mon Sep 17 00:00:00 2001 From: Ruben Romero Montes Date: Mon, 19 Feb 2024 15:32:47 +0100 Subject: [PATCH 1/2] fix: prevent empty cveId when setting aliases Signed-off-by: Ruben Romero Montes --- .../onguard/repository/redis/VulnerabilityRedisRepository.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/com/redhat/ecosystemappeng/onguard/repository/redis/VulnerabilityRedisRepository.java b/src/main/java/com/redhat/ecosystemappeng/onguard/repository/redis/VulnerabilityRedisRepository.java index 4cfca22..28fe9c0 100644 --- a/src/main/java/com/redhat/ecosystemappeng/onguard/repository/redis/VulnerabilityRedisRepository.java +++ b/src/main/java/com/redhat/ecosystemappeng/onguard/repository/redis/VulnerabilityRedisRepository.java @@ -80,7 +80,9 @@ public void save(Vulnerability vulnerability) { } public void setAliases(List aliases, String cveId) { + if(aliases != null && !aliases.isEmpty() && cveId != null) { aliasCommands.mset(aliases.stream().collect(Collectors.toMap(v -> Alias.getKey(v), v -> new Alias(v, cveId)))); + } } @Override From c0996e5c1b9f81e62a8bbc5f399a202d24138ed9 Mon Sep 17 00:00:00 2001 From: Ruben Romero Montes Date: Tue, 20 Feb 2024 18:21:04 +0100 Subject: [PATCH 2/2] fix: some invalid CVEs were loaded from the OSV database Signed-off-by: Ruben Romero Montes --- deploy/onguard-redis.yaml | 21 ++++++++- deploy/onguard.yaml | 45 +++++++++++++++++-- deploy/openshift/template.yaml | 43 ++++++++++++++++-- .../repository/VulnerabilityRepository.java | 2 + .../redis/VulnerabilityRedisRepository.java | 5 +++ .../onguard/rest/LoadRoute.java | 19 +++++--- .../onguard/service/LoadService.java | 6 +-- .../onguard/service/VulnerabilityService.java | 2 + .../service/VulnerabilityServiceImpl.java | 7 ++- .../onguard/service/nvd/NvdApi.java | 2 +- 10 files changed, 128 insertions(+), 24 deletions(-) diff --git a/deploy/onguard-redis.yaml b/deploy/onguard-redis.yaml index ec3e53b..75e5486 100644 --- a/deploy/onguard-redis.yaml +++ b/deploy/onguard-redis.yaml @@ -28,9 +28,14 @@ spec: volumeMounts: - name: logs mountPath: /redisinsight/logs + - name: data + mountPath: /data volumes: - name: logs emptyDir: {} + - name: data + persistentVolumeClaim: + claimName: redis-data --- apiVersion: v1 kind: Service @@ -44,9 +49,21 @@ spec: port: 6379 protocol: TCP targetPort: 6379 - - name: management + - name: http port: 8001 protocol: TCP targetPort: 8001 selector: - app: redis \ No newline at end of file + app: redis +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: redis-data +spec: + storageClassName: gp2 + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 4Gi \ No newline at end of file diff --git a/deploy/onguard.yaml b/deploy/onguard.yaml index 20cc01c..9b0b61a 100644 --- a/deploy/onguard.yaml +++ b/deploy/onguard.yaml @@ -89,7 +89,7 @@ data: if [ ${status_code} == '404' ]; then echo Load data is empty - curl -XPOST ${endpoint} + curl -s -XPOST ${endpoint} elif [ ${status_code} == '200' ]; then load=$(curl -s ${endpoint}) echo ${load} @@ -97,19 +97,31 @@ data: if [[ $status == "COMPLETED" ]]; then completed=$(echo "$load" | sed -E 's/.*"completed":"?([^,"]*)"?.*/\1/') echo "Completed load on ${completed}. Sync with most recent data" - curl -XPOST ${endpoint}?since=${completed} + curl -s -XPOST ${endpoint}?since=${completed} elif [[ $status == "COMPLETED_WITH_ERRORS" ]]; then echo "Previous load completed with errors. Resume" - curl -XPOST ${endpoint} + curl -s -XPOST ${endpoint} elif [[ $status == "PROCESSING" ]]; then echo "There is an ongoing load. Skip" else echo "Unknown load status ${load}. Force reload" - curl -XPOST ${endpoint}?reload=true + curl -s -XPOST ${endpoint}?reload=true fi else echo "Unexpected status code ${status_code}. Skip load and sync." fi + force_reload.sh: | + #!/bin/bash + set -euo pipefail + endpoint=http://onguard:9000/load + force_reload_date=2024-02-20T00:00:00 + + for i in {1..5}; do + echo retry $i + curl -s -XPOST ${endpoint}?reload_before=${force_reload_date} && exit 0 + sleep 1 + done + exit 1 --- apiVersion: batch/v1 kind: CronJob @@ -138,3 +150,28 @@ spec: defaultMode: 0777 restartPolicy: OnFailure +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: reload-nvd-db +spec: + template: + spec: + ttlSecondsAfterFinished: 3600 + containers: + - name: sync + image: registry.access.redhat.com/ubi9/ubi-minimal:9.3 + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - /scripts/force_reload.sh + volumeMounts: + - name: script-cm + mountPath: /scripts + volumes: + - name: script-cm + configMap: + name: sync-nvd-script + defaultMode: 0777 + restartPolicy: OnFailure \ No newline at end of file diff --git a/deploy/openshift/template.yaml b/deploy/openshift/template.yaml index ca83167..fd1bacd 100644 --- a/deploy/openshift/template.yaml +++ b/deploy/openshift/template.yaml @@ -131,7 +131,7 @@ objects: if [ ${status_code} == '404' ]; then echo Load data is empty - curl -XPOST ${endpoint} + curl -s -XPOST ${endpoint} elif [ ${status_code} == '200' ]; then load=$(curl -s ${endpoint}) echo ${load} @@ -139,19 +139,31 @@ objects: if [[ $status == "COMPLETED" ]]; then completed=$(echo "$load" | sed -E 's/.*"completed":"?([^,"]*)"?.*/\1/') echo "Completed load on ${completed}. Sync with most recent data" - curl -XPOST ${endpoint}?since=${completed} + curl -s -XPOST ${endpoint}?since=${completed} elif [[ $status == "COMPLETED_WITH_ERRORS" ]]; then echo "Previous load completed with errors. Resume" - curl -XPOST ${endpoint} + curl -s -XPOST ${endpoint} elif [[ $status == "PROCESSING" ]]; then echo "There is an ongoing load. Skip" else echo "Unknown load status ${load}. Force reload." - curl -XPOST ${endpoint}?reload=true + curl -s -XPOST ${endpoint}?reload=true fi else echo "Unexpected status code ${status_code}. Skip load and sync." fi + force_reload.sh: | + #!/bin/bash + set -euo pipefail + endpoint=http://onguard:9000/load + force_reload_date=2024-02-20T00:00:00 + + for i in {1..5}; do + echo retry $i + curl -s -XPOST ${endpoint}?reload_before=${force_reload_date} && exit 0 + sleep 1 + done + exit 1 - apiVersion: batch/v1 kind: CronJob metadata: @@ -178,6 +190,29 @@ objects: name: sync-nvd-script defaultMode: 0777 restartPolicy: OnFailure + - apiVersion: batch/v1 + kind: Job + metadata: + name: force-reload-nvd-db + spec: + template: + spec: + containers: + - name: sync + image: quay.io/app-sre/ubi9-ubi-minimal:9.3 + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - /scripts/force_reload.sh + volumeMounts: + - name: script-cm + mountPath: /scripts + volumes: + - name: script-cm + configMap: + name: sync-nvd-script + defaultMode: 0777 + restartPolicy: OnFailure parameters: - name: APP_NAME displayName: Application name diff --git a/src/main/java/com/redhat/ecosystemappeng/onguard/repository/VulnerabilityRepository.java b/src/main/java/com/redhat/ecosystemappeng/onguard/repository/VulnerabilityRepository.java index 72d5fd2..db7d45b 100644 --- a/src/main/java/com/redhat/ecosystemappeng/onguard/repository/VulnerabilityRepository.java +++ b/src/main/java/com/redhat/ecosystemappeng/onguard/repository/VulnerabilityRepository.java @@ -35,4 +35,6 @@ public interface VulnerabilityRepository { List listByAliases(List aliases); void setAliases(List aliases, String cveId); + + void removeAll(); } diff --git a/src/main/java/com/redhat/ecosystemappeng/onguard/repository/redis/VulnerabilityRedisRepository.java b/src/main/java/com/redhat/ecosystemappeng/onguard/repository/redis/VulnerabilityRedisRepository.java index 28fe9c0..b4f60e2 100644 --- a/src/main/java/com/redhat/ecosystemappeng/onguard/repository/redis/VulnerabilityRedisRepository.java +++ b/src/main/java/com/redhat/ecosystemappeng/onguard/repository/redis/VulnerabilityRedisRepository.java @@ -144,4 +144,9 @@ public List listByAliases(List aliasIds) { return results; } + @Override + public void removeAll() { + aliasCommands.getDataSource().flushall(); + } + } diff --git a/src/main/java/com/redhat/ecosystemappeng/onguard/rest/LoadRoute.java b/src/main/java/com/redhat/ecosystemappeng/onguard/rest/LoadRoute.java index 40c15bd..5c9788e 100644 --- a/src/main/java/com/redhat/ecosystemappeng/onguard/rest/LoadRoute.java +++ b/src/main/java/com/redhat/ecosystemappeng/onguard/rest/LoadRoute.java @@ -17,9 +17,9 @@ */ package com.redhat.ecosystemappeng.onguard.rest; -import java.text.ParseException; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -49,9 +49,11 @@ public class LoadRoute { public void registerManagementRoutes(@Observes ManagementInterface mi) { mi.router().post("/load").handler(ctx -> { final LocalDateTime since; + final LocalDateTime reloadBefore; try { since = parseDateParam(ctx.request().getParam("since")); - } catch (ParseException e) { + reloadBefore = parseDateParam(ctx.request().getParam("reload_before")); + } catch (DateTimeParseException e) { ctx.response().setStatusCode(400).setStatusMessage(e.getMessage()).end(); return; } @@ -59,12 +61,15 @@ public void registerManagementRoutes(@Observes ManagementInterface mi) { Uni.createFrom() .voidItem() .invoke(() -> { - if(reload) { + if(reloadBefore != null) { + var bulk = loadService.get(); + if(bulk != null && bulk.completed()!= null && reloadBefore.isBefore(bulk.completed())) { + loadService.restart(); + } + } else if(reload) { loadService.restart(); - } else { - loadService.loadFromNvdApi(since); } - + loadService.loadFromNvdApi(since); }) .runSubscriptionOn(Infrastructure.getDefaultExecutor()) .subscribeAsCompletionStage() @@ -107,7 +112,7 @@ public void registerManagementRoutes(@Observes ManagementInterface mi) { }); } - private LocalDateTime parseDateParam(String param) throws ParseException { + private LocalDateTime parseDateParam(String param) throws DateTimeParseException { if(param != null) { return LocalDateTime.parse(param, DateTimeFormatter.ISO_LOCAL_DATE_TIME); } diff --git a/src/main/java/com/redhat/ecosystemappeng/onguard/service/LoadService.java b/src/main/java/com/redhat/ecosystemappeng/onguard/service/LoadService.java index 970109b..db2da06 100644 --- a/src/main/java/com/redhat/ecosystemappeng/onguard/service/LoadService.java +++ b/src/main/java/com/redhat/ecosystemappeng/onguard/service/LoadService.java @@ -70,8 +70,6 @@ public void loadFromNvdApi(LocalDateTime since) { Multi.createFrom().items(vulnerabilities.stream()) .runSubscriptionOn(Infrastructure.getDefaultExecutor()).subscribe() .with(vulnerabilityService::ingestNvdVulnerability); - vulnerabilities.stream().parallel().forEach(vulnerabilityService::ingestNvdVulnerability); - synchronized (this) { bulk = bulkRepository.get(); var builder = Bulk.builder(bulk).index(bulk.index() + loaded).pageSize(pageSize); @@ -79,7 +77,6 @@ public void loadFromNvdApi(LocalDateTime since) { var completed = LocalDateTime.now(ZoneId.systemDefault()); builder.completed(completed) .status(Status.COMPLETED); - } bulkRepository.set(builder.build()); } @@ -114,8 +111,7 @@ public synchronized Bulk cancel() { } public void restart() { - bulkRepository.remove(); - loadFromNvdApi(null); + vulnerabilityService.cleanUp(); } } diff --git a/src/main/java/com/redhat/ecosystemappeng/onguard/service/VulnerabilityService.java b/src/main/java/com/redhat/ecosystemappeng/onguard/service/VulnerabilityService.java index 6592cbd..22d406b 100644 --- a/src/main/java/com/redhat/ecosystemappeng/onguard/service/VulnerabilityService.java +++ b/src/main/java/com/redhat/ecosystemappeng/onguard/service/VulnerabilityService.java @@ -32,4 +32,6 @@ public interface VulnerabilityService { void ingestNvdVulnerability(Vulnerability vuln); + void cleanUp(); + } diff --git a/src/main/java/com/redhat/ecosystemappeng/onguard/service/VulnerabilityServiceImpl.java b/src/main/java/com/redhat/ecosystemappeng/onguard/service/VulnerabilityServiceImpl.java index f687460..8ff4505 100644 --- a/src/main/java/com/redhat/ecosystemappeng/onguard/service/VulnerabilityServiceImpl.java +++ b/src/main/java/com/redhat/ecosystemappeng/onguard/service/VulnerabilityServiceImpl.java @@ -97,7 +97,7 @@ public Map> findByPurls(List purls, boolean @Override public void ingestNvdVulnerability(Vulnerability vuln) { - if (vuln == null) { + if (vuln == null || !OsvVulnerability.CVE_PATTERN.matcher(vuln.cveId()).matches()) { return; } LOGGER.debug("Ingest NVD Vulnerability {}", vuln.cveId()); @@ -161,4 +161,9 @@ private Vulnerability getOsvVulnerabilityData(String alias) { .affected(osvVuln.affected()).build(); } + @Override + public void cleanUp() { + repository.removeAll(); + } + } diff --git a/src/main/java/com/redhat/ecosystemappeng/onguard/service/nvd/NvdApi.java b/src/main/java/com/redhat/ecosystemappeng/onguard/service/nvd/NvdApi.java index 0b0ad8a..a4866e7 100644 --- a/src/main/java/com/redhat/ecosystemappeng/onguard/service/nvd/NvdApi.java +++ b/src/main/java/com/redhat/ecosystemappeng/onguard/service/nvd/NvdApi.java @@ -39,7 +39,7 @@ public interface NvdApi { static final long NVD_API_WINDOW_SECS = 30; - static final String SIMPLE_ISO_8601 = "yyyy-MM-dd"; + @GET @ClientHeaderParam(name = "apiKey", value = "${api.nvd.apikey}") @Produces(MediaType.APPLICATION_JSON)