diff --git a/.github/workflows/grype-vulnerability-scanner.yaml b/.github/workflows/grype-vulnerability-scanner.yaml index 986a29ea30..6589cf135c 100644 --- a/.github/workflows/grype-vulnerability-scanner.yaml +++ b/.github/workflows/grype-vulnerability-scanner.yaml @@ -34,10 +34,10 @@ jobs: steps: - name: Printing Image Registry id: image-registry - run: echo "image_registry=${{fromJson(needs.vulnerability-scanner.outputs.valid_images).image_registry}}" >> "$GITHUB_ENV" + run: echo "image_registry=${{fromJson(needs.vulnerability-scanner.outputs.valid_images).image_registry}}" >> "$GITHUB_ENV" - name: Printing Image Tag id: image-tag - run: echo "image_tag=${{fromJson(needs.vulnerability-scanner.outputs.valid_images).tag}}" >> "$GITHUB_ENV" + run: echo "image_tag=${{fromJson(needs.vulnerability-scanner.outputs.valid_images).tag}}" >> "$GITHUB_ENV" - name: Printing Image Path run: echo "image_path=${{env.image_registry}}/${{matrix.images}}:${{env.image_tag}}" >> "$GITHUB_ENV" - name: Running vulnerability scanner @@ -55,6 +55,6 @@ jobs: with: ref: master path: repo - - name: Parsing vulnerability scanner report - run: go run repo/pkg/tools/grype_report_parser_tool.go -s "High,Critical" -p results.json + - name: Parsing vulnerability scanner report + run: go run repo/pkg/tools/grype_report_parser_tool.go -s "High,Critical" -p results.json --github diff --git a/.goreleaser.yml b/.goreleaser.yml index 180b1cd013..ccfb9f239e 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -74,6 +74,8 @@ dockers: image_templates: - 'ghcr.io/kanisterio/postgres-kanister-tools:{{ .Tag }}' dockerfile: 'docker/postgres-kanister-tools/Dockerfile' + build_flag_templates: + - "--build-arg=TOOLS_IMAGE=ghcr.io/kanisterio/kanister-tools:{{ .Tag }}" - image_templates: - 'ghcr.io/kanisterio/postgresql:{{ .Tag }}' dockerfile: 'docker/postgresql/Dockerfile' @@ -101,11 +103,15 @@ dockers: image_templates: - 'ghcr.io/kanisterio/mongodb:{{ .Tag }}' dockerfile: 'docker/mongodb/Dockerfile' + build_flag_templates: + - "--build-arg=TOOLS_IMAGE=ghcr.io/kanisterio/kanister-tools:{{ .Tag }}" - ids: - kando image_templates: - 'ghcr.io/kanisterio/cassandra:{{ .Tag }}' dockerfile: 'docker/cassandra/Dockerfile' + build_flag_templates: + - "--build-arg=TOOLS_IMAGE=ghcr.io/kanisterio/kanister-tools:{{ .Tag }}" - image_templates: - 'ghcr.io/kanisterio/kafka-adobe-s3-source-connector:{{ .Tag }}' dockerfile: 'docker/kafka-adobes3Connector/image/adobeSource.Dockerfile' diff --git a/docker/cassandra/Dockerfile b/docker/cassandra/Dockerfile index f785eb26d9..a782b289a3 100644 --- a/docker/cassandra/Dockerfile +++ b/docker/cassandra/Dockerfile @@ -1,9 +1,17 @@ -FROM bitnami/cassandra:3.11.8-debian-10-r20 +# We get tools from tools image +# Tools are not up to date in debian repos +ARG TOOLS_IMAGE +FROM ${TOOLS_IMAGE} AS TOOLS_IMAGE + +# Actual image base +FROM bitnami/cassandra:4.1.3-debian-11-r76 MAINTAINER "Tom Manville " # Install restic to take backups -COPY --from=restic/restic:0.11.0 /usr/bin/restic /usr/local/bin/restic +COPY --from=TOOLS_IMAGE /usr/local/bin/restic /usr/local/bin/restic +# Update gosu from recent version +COPY --from=TOOLS_IMAGE /usr/local/bin/gosu /usr/local/bin/gosu -# Install kando +# Install kando ADD kando /usr/local/bin/ diff --git a/docker/kafka-adobes3Connector/image/adobeSink.Dockerfile b/docker/kafka-adobes3Connector/image/adobeSink.Dockerfile index edb2a886fe..194fcaa4e2 100644 --- a/docker/kafka-adobes3Connector/image/adobeSink.Dockerfile +++ b/docker/kafka-adobes3Connector/image/adobeSink.Dockerfile @@ -1,11 +1,10 @@ -FROM confluentinc/cp-kafka-connect:6.1.0 +FROM confluentinc/cp-kafka-connect:7.4.3 USER root -RUN microdnf install -y lsof - -# copy the jar files +RUN microdnf install -y lsof platform-python python3-libs +# TODO: maybe use builder image for that RUN microdnf install -y \ java-1.8.0-openjdk \ java-1.8.0-openjdk-devel @@ -14,9 +13,14 @@ ENV JAVA_HOME /usr/lib/jvm/java-1.8.0-openjdk/ RUN microdnf install git -y RUN java -version RUN git clone https://github.com/adobe/kafka-connect-s3.git +# Temp patch until vulnerable deps are fixed +RUN sed -i "s/versions.awsSdkS3 = '1.11.803'/versions.awsSdkS3 = '1.12.261'/g" kafka-connect-s3/dependencies.gradle +RUN sed -i "s/versions.jackson = '2.10.4'/versions.jackson = '2.12.7.1'/g" kafka-connect-s3/dependencies.gradle RUN cd kafka-connect-s3 && ./gradlew shadowJar # copy the jar files RUN cp ./kafka-connect-s3/build/libs/kafka-connect-s3-chart/kafka-connect/0.0.4-2a8a4aa-all.jar /opt/ +# cleanup +RUN rm -rf ~/.gradle ./kafka-connect-s3 # Install kando ADD kando /usr/local/bin/ diff --git a/docker/kafka-adobes3Connector/image/adobeSource.Dockerfile b/docker/kafka-adobes3Connector/image/adobeSource.Dockerfile index 9532d85b8e..ffb5fd97f3 100644 --- a/docker/kafka-adobes3Connector/image/adobeSource.Dockerfile +++ b/docker/kafka-adobes3Connector/image/adobeSource.Dockerfile @@ -1,8 +1,11 @@ -FROM confluentinc/cp-kafka-connect:6.1.0 +FROM confluentinc/cp-kafka-connect:7.4.3 USER root -# copy the jar files +RUN microdnf install -y \ + platform-python python3-libs + +# TODO: maybe use builder image for that RUN microdnf install -y \ java-1.8.0-openjdk \ java-1.8.0-openjdk-devel @@ -11,9 +14,14 @@ ENV JAVA_HOME /usr/lib/jvm/java-1.8.0-openjdk/ RUN microdnf install git -y RUN java -version RUN git clone https://github.com/adobe/kafka-connect-s3.git +# Temp patch until vulnerable deps are fixed +RUN sed -i "s/versions.awsSdkS3 = '1.11.803'/versions.awsSdkS3 = '1.12.261'/g" kafka-connect-s3/dependencies.gradle +RUN sed -i "s/versions.jackson = '2.10.4'/versions.jackson = '2.12.7.1'/g" kafka-connect-s3/dependencies.gradle RUN cd kafka-connect-s3 && ./gradlew shadowJar - +# copy the jar files RUN cp ./kafka-connect-s3/build/libs/kafka-connect-s3-chart/kafka-connect/0.0.4-2a8a4aa-all.jar /opt/ +# cleanup +RUN rm -rf ~/.gradle ./kafka-connect-s3 # adding script to monitor source connector COPY docker/kafka-adobes3Connector/image/adobe-monitorsource.sh monitorconnect.sh diff --git a/docker/kanister-elasticsearch/image/Dockerfile b/docker/kanister-elasticsearch/image/Dockerfile index 428927ffbd..fb9eccb53e 100644 --- a/docker/kanister-elasticsearch/image/Dockerfile +++ b/docker/kanister-elasticsearch/image/Dockerfile @@ -1,3 +1,5 @@ +# We get tools from tools image +# Tools are not up to date in debian repos ARG TOOLS_IMAGE FROM ${TOOLS_IMAGE} AS TOOLS_IMAGE @@ -10,7 +12,7 @@ RUN apt update RUN apt install -y npm bash curl libcap2-bin RUN curl -fsSL https://deb.nodesource.com/setup_current.x | bash - && \ apt-get install -y nodejs -RUN npm install -g npm yo grunt-cli bower express +RUN npm install -g npm RUN npm install elasticdump -g RUN setcap cap_chown,cap_fowner,cap_dac_override+iep /usr/local/bin/kopia diff --git a/docker/mongodb/Dockerfile b/docker/mongodb/Dockerfile index e053d59368..3172061f54 100755 --- a/docker/mongodb/Dockerfile +++ b/docker/mongodb/Dockerfile @@ -1,6 +1,14 @@ -FROM bitnami/mongodb:5.0.14-debian-11-r0 +# We get tools from tools image +# Tools are not up to date in debian repos +ARG TOOLS_IMAGE +FROM ${TOOLS_IMAGE} AS TOOLS_IMAGE + +FROM bitnami/mongodb:7.0.4-debian-11-r0 LABEL maintainer="Tom Manville " +# Update gosu from recent version +COPY --from=TOOLS_IMAGE /usr/local/bin/gosu /usr/local/bin/gosu + # Install kando ADD kando /usr/local/bin/ diff --git a/docker/postgres-kanister-tools/Dockerfile b/docker/postgres-kanister-tools/Dockerfile index 8cdb6ab51e..bcf9e225c9 100644 --- a/docker/postgres-kanister-tools/Dockerfile +++ b/docker/postgres-kanister-tools/Dockerfile @@ -1,4 +1,10 @@ -FROM postgres:16-bullseye +# We get tools from tools image +# Tools are not up to date in debian repos +ARG TOOLS_IMAGE +FROM ${TOOLS_IMAGE} AS TOOLS_IMAGE + +# Actual image base +FROM postgres:16.1-bullseye ENV DEBIAN_FRONTEND noninteractive @@ -9,7 +15,11 @@ RUN apt-get update && apt-get -y install curl python3 groff less jq python3-pip pip3 install --upgrade awscli && \ apt-get clean -COPY --from=restic/restic:0.11.0 /usr/bin/restic /usr/local/bin/restic +# Install restic to take backups +COPY --from=TOOLS_IMAGE /usr/local/bin/restic /usr/local/bin/restic +# Update gosu from recent version +COPY --from=TOOLS_IMAGE /usr/local/bin/gosu /usr/local/bin/gosu + ADD kando /usr/local/bin/ CMD ["tail", "-f", "/dev/null"] diff --git a/docker/tools/Dockerfile b/docker/tools/Dockerfile index df57da72a1..e5e79c0c14 100644 --- a/docker/tools/Dockerfile +++ b/docker/tools/Dockerfile @@ -3,6 +3,8 @@ FROM golang:1.21-bullseye AS builder ARG kopia_build_commit=master ARG kopia_repo_org=kopia +ARG restic_vsn=v0.16.2 +ARG gosu_vsn=1.17 ENV CGO_ENABLED=1 GOEXPERIMENT=boringcrypto GO_EXTLINK_ENABLED=0 RUN apt-get install git @@ -17,12 +19,25 @@ ENV GITHUB_REPOSITORY=https://github.com/restic/restic WORKDIR /restic -RUN git checkout v0.16.2 && \ +RUN git checkout ${restic_vsn} && \ echo 'package main' > cmd/restic/fipsonly.go && \ echo 'import _ "crypto/tls/fipsonly"' >> cmd/restic/fipsonly.go # use debug flag to preserve symbols RUN go run build.go --tags debug +# Build restic binary from source - released version +# This will allow us to bring in security fixes more up to date then apt repos +WORKDIR / + +RUN git clone https://github.com/tianon/gosu.git + +ENV GITHUB_REPOSITORY=https://github.com/tianon/gosu + +WORKDIR /gosu + +RUN git checkout ${gosu_vsn} +RUN go build -o gosu + # Build kopia binary from specific commit WORKDIR / @@ -70,6 +85,7 @@ LABEL name="kanister-tools" \ description="Tools for application-specific data protection" COPY --from=builder /restic/restic /usr/local/bin/restic +COPY --from=builder /gosu/gosu /usr/local/bin/gosu COPY --from=builder /kopia/kopia /usr/local/bin/kopia COPY LICENSE /licenses/LICENSE diff --git a/examples/cassandra/README.md b/examples/cassandra/README.md index 6f956bc6b5..a71e749cac 100644 --- a/examples/cassandra/README.md +++ b/examples/cassandra/README.md @@ -33,7 +33,7 @@ $ helm install cassandra bitnami/cassandra --namespace --set ima ``` -This command will install Cassandra on your Kubernetes cluster with 2 nodes. You can notice that we are using custom image of Cassandra in the helm to install the Cassandra cluster. The reason is we have to use some Kanister tools to take backup, so only change that we have done is including that tooling on top of standard `cassandra:3.11.8-debian-10-r20` image. +This command will install Cassandra on your Kubernetes cluster with 2 nodes. You can notice that we are using custom image of Cassandra in the helm to install the Cassandra cluster. The reason is we have to use some Kanister tools to take backup, so only change that we have done is including that tooling on top of standard `4.1.3-debian-11-r76` image. ## Integrating with Kanister diff --git a/pkg/tools/grype_report_parser_tool.go b/pkg/tools/grype_report_parser_tool.go index 5aebeedf95..39ebf40328 100644 --- a/pkg/tools/grype_report_parser_tool.go +++ b/pkg/tools/grype_report_parser_tool.go @@ -17,6 +17,7 @@ type vulnerabilityScannerResponse struct { type matchResponse struct { Vulnerabilities vulnerabilityReport `json:"vulnerability"` + Artifact artifact `json:"artifact"` } type fixVersionsResponse struct { @@ -26,46 +27,55 @@ type fixVersionsResponse struct { type vulnerabilityReport struct { ID string `json:"id"` + DataSource string `json:"dataSource,omitempty"` Severity string `json:"severity"` Namespace string `json:"namespace"` Description string `json:"description"` FixVersions fixVersionsResponse `json:"fix"` } +type artifact struct { + Name string `json:"name"` + Version string `json:"version"` + Type string `json:"type"` + Purl string `json:"purl"` + Locations json.RawMessage `json:"locations,omitempty"` + Metadata json.RawMessage `json:"metadata,omitempty"` +} + // filterVulnerabilityReportMatches filters vulnerabilities based on the severity levels set in severityTypeSet -func filterVulnerabilityReportMatches(matches []matchResponse, severityTypeSet map[string]bool) ([]vulnerabilityReport, error) { - mv := make([]vulnerabilityReport, 0) +func filterVulnerabilityReportMatches(matches []matchResponse, severityTypeSet map[string]bool) ([]matchResponse, error) { + filtered := make([]matchResponse, 0) for _, m := range matches { if severityTypeSet[m.Vulnerabilities.Severity] { - mv = append(mv, m.Vulnerabilities) + filtered = append(filtered, m) } } - return mv, nil + return filtered, nil } // decodeVulnerabilityReports unmarshals the specific matches from the vulnerability report // and returns a list of vulnerabilities based on the severity levels set in severityTypeSet -func decodeVulnerabilityReports(v vulnerabilityScannerResponse, severityTypeSet map[string]bool) ([]vulnerabilityReport, error) { +func decodeVulnerabilityReports(v vulnerabilityScannerResponse, severityTypeSet map[string]bool) ([]matchResponse, error) { var mr []matchResponse - mv := make([]vulnerabilityReport, 0) if err := json.Unmarshal(v.Matches, &mr); err != nil { - return mv, fmt.Errorf("failed to unmarshal matches: %v", err) + return make([]matchResponse, 0), fmt.Errorf("failed to unmarshal matches: %v", err) } return filterVulnerabilityReportMatches(mr, severityTypeSet) } // parseVulerabilitiesReport unmarshals the vulnerability report and returns a list of vulnerabilities // based on the severity levels set in severityTypeSet -func parseVulerabilitiesReport(filePath string, severityLevels []string) ([]vulnerabilityReport, error) { - mv := make([]vulnerabilityReport, 0) +func parseVulerabilitiesReport(filePath string, severityLevels []string) ([]matchResponse, error) { + mr := make([]matchResponse, 0) data, err := os.ReadFile(filePath) if err != nil { - return mv, fmt.Errorf("failed to read file at path %s: %v", filePath, err) + return mr, fmt.Errorf("failed to read file at path %s: %v", filePath, err) } var response vulnerabilityScannerResponse if err = json.Unmarshal(data, &response); err != nil { - return mv, fmt.Errorf("failed to unmarshal response: %v", err) + return mr, fmt.Errorf("failed to unmarshal response: %v", err) } severityTypeSet := make(map[string]bool) for _, severityLevel := range severityLevels { @@ -75,13 +85,30 @@ func parseVulerabilitiesReport(filePath string, severityLevels []string) ([]vuln } // printResult Displays the filtered list of vulnerability reports to stdout -func printResult(mv []vulnerabilityReport) { - for _, vulnerability := range mv { - fmt.Printf("ID: %s\n", vulnerability.ID) - fmt.Printf("Severity: %s\n", vulnerability.Severity) - fmt.Printf("Namespace: %s\n", vulnerability.Namespace) - fmt.Printf("Description: %s\n", vulnerability.Description) - fmt.Printf("Fix Versions: %v\n", vulnerability.FixVersions) +func printResult(mr []matchResponse, githubActionOutput bool) { + for _, response := range mr { + fmt.Printf("ID: %s\n", response.Vulnerabilities.ID) + fmt.Printf("Link: https://github.com/advisories/%s\n", response.Vulnerabilities.DataSource) + fmt.Printf("Severity: %s\n", response.Vulnerabilities.Severity) + fmt.Printf("Namespace: %s\n", response.Vulnerabilities.Namespace) + fmt.Printf("Description: %s\n", response.Vulnerabilities.Description) + fmt.Printf("Fix Versions: %v\n", response.Vulnerabilities.FixVersions) + fmt.Println("Package:") + fmt.Printf("Name: %v\n", response.Artifact.Name) + fmt.Printf("Version: %v\n", response.Artifact.Version) + fmt.Printf("Type: %v\n", response.Artifact.Type) + fmt.Printf("PURL: %v\n", response.Artifact.Purl) + if githubActionOutput { + fmt.Println("::group::Locations") + fmt.Printf("%s\n", response.Artifact.Locations) + fmt.Println("::endgroup::") + fmt.Println("::group::Metadata") + fmt.Printf("%s\n", response.Artifact.Metadata) + fmt.Println("::endgroup::") + } else { + fmt.Printf("Locations: %s\n", response.Artifact.Locations) + fmt.Printf("Metadata: \n%s\n", response.Artifact.Metadata) + } fmt.Printf("\n") } } @@ -90,6 +117,7 @@ func main() { validSeverityLevels := []string{"Negliable", "Low", "Medium", "High", "Critical"} severityInputList := flag.String("s", "High,Critical", "Comma separated list of severity levels to scan. Valid severity levels are: "+strings.Join(validSeverityLevels, ",")) reportJsonFilePath := flag.String("p", "", "Path to the JSON file containing the vulnerabilities report") + githubActionOutput := flag.Bool("github", false, "Whether to use github action output format") flag.Parse() // passing file path is compulsory @@ -98,15 +126,15 @@ func main() { os.Exit(1) } severityLevels := strings.Split(*severityInputList, ",") - mv, err := parseVulerabilitiesReport(*reportJsonFilePath, severityLevels) + mr, err := parseVulerabilitiesReport(*reportJsonFilePath, severityLevels) if err != nil { fmt.Printf("Failed to parse vulnerabilities report: %v\n", err) os.Exit(1) } - fmt.Printf("Found %d vulnerabilities\n", len(mv)) - if len(mv) == 0 { + fmt.Printf("Found %d vulnerabilities\n", len(mr)) + if len(mr) == 0 { os.Exit(0) } - printResult(mv) + printResult(mr, *githubActionOutput) os.Exit(1) } diff --git a/pkg/tools/grype_report_parser_tool_test.go b/pkg/tools/grype_report_parser_tool_test.go index 20645538e6..7b8bb84e10 100644 --- a/pkg/tools/grype_report_parser_tool_test.go +++ b/pkg/tools/grype_report_parser_tool_test.go @@ -48,7 +48,7 @@ func (v *VulnerabilityParserSuite) TestValidJsonForMatchingVulerabilities(c *C) c.Assert(len(matchingVulnerabilities), Equals, 2) c.Assert(err, IsNil) for index, vulnerability := range matchingVulnerabilities { - c.Assert(vulnerability.ID, Equals, expectedIds[index]) - c.Assert(vulnerability.Severity, Equals, severityLevels[index]) + c.Assert(vulnerability.Vulnerabilities.ID, Equals, expectedIds[index]) + c.Assert(vulnerability.Vulnerabilities.Severity, Equals, severityLevels[index]) } }