Skip to content

Commit

Permalink
Update blueprint images dependencies to fix reported vulnerabilities
Browse files Browse the repository at this point in the history
Update base images
Build go libs in kanister-tools image
Remove unused tools with vulnerabilities

Expand vulnerability report tool output to show more information
  • Loading branch information
hairyhum committed Dec 11, 2023
1 parent 7a0873a commit d2f3c33
Show file tree
Hide file tree
Showing 12 changed files with 134 additions and 44 deletions.
8 changes: 4 additions & 4 deletions .github/workflows/grype-vulnerability-scanner.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,10 @@ jobs:
steps:
- name: Printing Image Registry
id: image-registry
run: echo "image_registry=${{fromJson(needs.vulnerability-scanner.outputs.valid_images).image_registry}}" >> "$GITHUB_ENV"
run: echo "image_registry=${{fromJson(needs.vulnerability-scanner.outputs.valid_images).image_registry}}" >> "$GITHUB_ENV"
- name: Printing Image Tag
id: image-tag
run: echo "image_tag=${{fromJson(needs.vulnerability-scanner.outputs.valid_images).tag}}" >> "$GITHUB_ENV"
run: echo "image_tag=${{fromJson(needs.vulnerability-scanner.outputs.valid_images).tag}}" >> "$GITHUB_ENV"
- name: Printing Image Path
run: echo "image_path=${{env.image_registry}}/${{matrix.images}}:${{env.image_tag}}" >> "$GITHUB_ENV"
- name: Running vulnerability scanner
Expand All @@ -55,6 +55,6 @@ jobs:
with:
ref: master
path: repo
- name: Parsing vulnerability scanner report
run: go run repo/pkg/tools/grype_report_parser_tool.go -s "High,Critical" -p results.json
- name: Parsing vulnerability scanner report
run: go run repo/pkg/tools/grype_report_parser_tool.go -s "High,Critical" -p results.json --github

6 changes: 6 additions & 0 deletions .goreleaser.yml
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,8 @@ dockers:
image_templates:
- 'ghcr.io/kanisterio/postgres-kanister-tools:{{ .Tag }}'
dockerfile: 'docker/postgres-kanister-tools/Dockerfile'
build_flag_templates:
- "--build-arg=TOOLS_IMAGE=ghcr.io/kanisterio/kanister-tools:{{ .Tag }}"
- image_templates:
- 'ghcr.io/kanisterio/postgresql:{{ .Tag }}'
dockerfile: 'docker/postgresql/Dockerfile'
Expand Down Expand Up @@ -99,11 +101,15 @@ dockers:
image_templates:
- 'ghcr.io/kanisterio/mongodb:{{ .Tag }}'
dockerfile: 'docker/mongodb/Dockerfile'
build_flag_templates:
- "--build-arg=TOOLS_IMAGE=ghcr.io/kanisterio/kanister-tools:{{ .Tag }}"
- ids:
- kando
image_templates:
- 'ghcr.io/kanisterio/cassandra:{{ .Tag }}'
dockerfile: 'docker/cassandra/Dockerfile'
build_flag_templates:
- "--build-arg=TOOLS_IMAGE=ghcr.io/kanisterio/kanister-tools:{{ .Tag }}"
- image_templates:
- 'ghcr.io/kanisterio/kafka-adobe-s3-source-connector:{{ .Tag }}'
dockerfile: 'docker/kafka-adobes3Connector/image/adobeSource.Dockerfile'
Expand Down
14 changes: 11 additions & 3 deletions docker/cassandra/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,9 +1,17 @@
FROM bitnami/cassandra:3.11.8-debian-10-r20
# We get tools from tools image
# Tools are not up to date in debian repos
ARG TOOLS_IMAGE
FROM ${TOOLS_IMAGE} AS TOOLS_IMAGE

# Actual image base
FROM bitnami/cassandra:4.1.3-debian-11-r76

MAINTAINER "Tom Manville <tom@kasten.io>"

# Install restic to take backups
COPY --from=restic/restic:0.11.0 /usr/bin/restic /usr/local/bin/restic
COPY --from=TOOLS_IMAGE /usr/local/bin/restic /usr/local/bin/restic
# Update gosu from recent version
COPY --from=TOOLS_IMAGE /usr/local/bin/gosu /usr/local/bin/gosu

# Install kando
# Install kando
ADD kando /usr/local/bin/
12 changes: 8 additions & 4 deletions docker/kafka-adobes3Connector/image/adobeSink.Dockerfile
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
FROM confluentinc/cp-kafka-connect:6.1.0
FROM confluentinc/cp-kafka-connect:7.4.3

USER root

RUN microdnf install -y lsof

# copy the jar files
RUN microdnf install -y lsof platform-python python3-libs

# TODO: maybe use builder image for that
RUN microdnf install -y \
java-1.8.0-openjdk \
java-1.8.0-openjdk-devel
Expand All @@ -14,9 +13,14 @@ ENV JAVA_HOME /usr/lib/jvm/java-1.8.0-openjdk/
RUN microdnf install git -y
RUN java -version
RUN git clone https://github.com/adobe/kafka-connect-s3.git
# Temp patch until vulnerable deps are fixed
RUN sed -i "s/versions.awsSdkS3 = '1.11.803'/versions.awsSdkS3 = '1.12.261'/g" kafka-connect-s3/dependencies.gradle
RUN sed -i "s/versions.jackson = '2.10.4'/versions.jackson = '2.12.7.1'/g" kafka-connect-s3/dependencies.gradle
RUN cd kafka-connect-s3 && ./gradlew shadowJar
# copy the jar files
RUN cp ./kafka-connect-s3/build/libs/kafka-connect-s3-chart/kafka-connect/0.0.4-2a8a4aa-all.jar /opt/
# cleanup
RUN rm -rf ~/.gradle ./kafka-connect-s3

# Install kando
ADD kando /usr/local/bin/
Expand Down
14 changes: 11 additions & 3 deletions docker/kafka-adobes3Connector/image/adobeSource.Dockerfile
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
FROM confluentinc/cp-kafka-connect:6.1.0
FROM confluentinc/cp-kafka-connect:7.4.3

USER root
# copy the jar files

RUN microdnf install -y \
platform-python python3-libs

# TODO: maybe use builder image for that
RUN microdnf install -y \
java-1.8.0-openjdk \
java-1.8.0-openjdk-devel
Expand All @@ -11,9 +14,14 @@ ENV JAVA_HOME /usr/lib/jvm/java-1.8.0-openjdk/
RUN microdnf install git -y
RUN java -version
RUN git clone https://github.com/adobe/kafka-connect-s3.git
# Temp patch until vulnerable deps are fixed
RUN sed -i "s/versions.awsSdkS3 = '1.11.803'/versions.awsSdkS3 = '1.12.261'/g" kafka-connect-s3/dependencies.gradle
RUN sed -i "s/versions.jackson = '2.10.4'/versions.jackson = '2.12.7.1'/g" kafka-connect-s3/dependencies.gradle
RUN cd kafka-connect-s3 && ./gradlew shadowJar

# copy the jar files
RUN cp ./kafka-connect-s3/build/libs/kafka-connect-s3-chart/kafka-connect/0.0.4-2a8a4aa-all.jar /opt/
# cleanup
RUN rm -rf ~/.gradle ./kafka-connect-s3

# adding script to monitor source connector
COPY docker/kafka-adobes3Connector/image/adobe-monitorsource.sh monitorconnect.sh
Expand Down
4 changes: 3 additions & 1 deletion docker/kanister-elasticsearch/image/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# We get tools from tools image
# Tools are not up to date in debian repos
ARG TOOLS_IMAGE
FROM ${TOOLS_IMAGE} AS TOOLS_IMAGE

Expand All @@ -10,7 +12,7 @@ RUN apt update
RUN apt install -y npm bash curl libcap2-bin
RUN curl -fsSL https://deb.nodesource.com/setup_current.x | bash - && \
apt-get install -y nodejs
RUN npm install -g npm yo grunt-cli bower express
RUN npm install -g npm
RUN npm install elasticdump -g

RUN setcap cap_chown,cap_fowner,cap_dac_override+iep /usr/local/bin/kopia
Expand Down
10 changes: 9 additions & 1 deletion docker/mongodb/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,6 +1,14 @@
FROM bitnami/mongodb:5.0.14-debian-11-r0
# We get tools from tools image
# Tools are not up to date in debian repos
ARG TOOLS_IMAGE
FROM ${TOOLS_IMAGE} AS TOOLS_IMAGE

FROM bitnami/mongodb:7.0.4-debian-11-r0

LABEL maintainer="Tom Manville <tom@kasten.io>"

# Update gosu from recent version
COPY --from=TOOLS_IMAGE /usr/local/bin/gosu /usr/local/bin/gosu

# Install kando
ADD kando /usr/local/bin/
14 changes: 12 additions & 2 deletions docker/postgres-kanister-tools/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,10 @@
FROM postgres:16-bullseye
# We get tools from tools image
# Tools are not up to date in debian repos
ARG TOOLS_IMAGE
FROM ${TOOLS_IMAGE} AS TOOLS_IMAGE

# Actual image base
FROM postgres:16.1-bullseye

ENV DEBIAN_FRONTEND noninteractive

Expand All @@ -9,7 +15,11 @@ RUN apt-get update && apt-get -y install curl python3 groff less jq python3-pip
pip3 install --upgrade awscli && \
apt-get clean

COPY --from=restic/restic:0.11.0 /usr/bin/restic /usr/local/bin/restic
# Install restic to take backups
COPY --from=TOOLS_IMAGE /usr/local/bin/restic /usr/local/bin/restic
# Update gosu from recent version
COPY --from=TOOLS_IMAGE /usr/local/bin/gosu /usr/local/bin/gosu

ADD kando /usr/local/bin/

CMD ["tail", "-f", "/dev/null"]
18 changes: 17 additions & 1 deletion docker/tools/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ FROM golang:1.21-bullseye AS builder

ARG kopia_build_commit=master
ARG kopia_repo_org=kopia
ARG restic_vsn=v0.16.2
ARG gosu_vsn=1.17
ENV CGO_ENABLED=1 GOEXPERIMENT=boringcrypto GO_EXTLINK_ENABLED=0
RUN apt-get install git

Expand All @@ -17,9 +19,22 @@ ENV GITHUB_REPOSITORY=https://github.com/restic/restic

WORKDIR /restic

RUN git checkout v0.16.2
RUN git checkout ${restic_vsn}
RUN go run build.go

# Build restic binary from source - released version
# This will allow us to bring in security fixes more up to date then apt repos
WORKDIR /

RUN git clone https://github.com/tianon/gosu.git

ENV GITHUB_REPOSITORY=https://github.com/tianon/gosu

WORKDIR /gosu

RUN git checkout ${gosu_vsn}
RUN go build -o gosu

# Build kopia binary from specific commit
WORKDIR /

Expand Down Expand Up @@ -65,6 +80,7 @@ LABEL name="kanister-tools" \
description="Tools for application-specific data protection"

COPY --from=builder /restic/restic /usr/local/bin/restic
COPY --from=builder /gosu/gosu /usr/local/bin/gosu
COPY --from=builder /kopia/kopia /usr/local/bin/kopia
COPY LICENSE /licenses/LICENSE

Expand Down
2 changes: 1 addition & 1 deletion examples/cassandra/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ $ helm install cassandra bitnami/cassandra --namespace <app-namespace> --set ima


```
This command will install Cassandra on your Kubernetes cluster with 2 nodes. You can notice that we are using custom image of Cassandra in the helm to install the Cassandra cluster. The reason is we have to use some Kanister tools to take backup, so only change that we have done is including that tooling on top of standard `cassandra:3.11.8-debian-10-r20` image.
This command will install Cassandra on your Kubernetes cluster with 2 nodes. You can notice that we are using custom image of Cassandra in the helm to install the Cassandra cluster. The reason is we have to use some Kanister tools to take backup, so only change that we have done is including that tooling on top of standard `4.1.3-debian-11-r76` image.

## Integrating with Kanister

Expand Down
72 changes: 50 additions & 22 deletions pkg/tools/grype_report_parser_tool.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ type vulnerabilityScannerResponse struct {

type matchResponse struct {
Vulnerabilities vulnerabilityReport `json:"vulnerability"`
Artifact artifact `json:"artifact"`
}

type fixVersionsResponse struct {
Expand All @@ -26,46 +27,55 @@ type fixVersionsResponse struct {

type vulnerabilityReport struct {
ID string `json:"id"`
DataSource string `json:"dataSource,omitempty"`
Severity string `json:"severity"`
Namespace string `json:"namespace"`
Description string `json:"description"`
FixVersions fixVersionsResponse `json:"fix"`
}

type artifact struct {
Name string `json:"name"`
Version string `json:"version"`
Type string `json:"type"`
Purl string `json:"purl"`
Locations json.RawMessage `json:"locations,omitempty"`
Metadata json.RawMessage `json:"metadata,omitempty"`
}

// filterVulnerabilityReportMatches filters vulnerabilities based on the severity levels set in severityTypeSet
func filterVulnerabilityReportMatches(matches []matchResponse, severityTypeSet map[string]bool) ([]vulnerabilityReport, error) {
mv := make([]vulnerabilityReport, 0)
func filterVulnerabilityReportMatches(matches []matchResponse, severityTypeSet map[string]bool) ([]matchResponse, error) {
filtered := make([]matchResponse, 0)
for _, m := range matches {
if severityTypeSet[m.Vulnerabilities.Severity] {
mv = append(mv, m.Vulnerabilities)
filtered = append(filtered, m)
}
}
return mv, nil
return filtered, nil
}

// decodeVulnerabilityReports unmarshals the specific matches from the vulnerability report
// and returns a list of vulnerabilities based on the severity levels set in severityTypeSet
func decodeVulnerabilityReports(v vulnerabilityScannerResponse, severityTypeSet map[string]bool) ([]vulnerabilityReport, error) {
func decodeVulnerabilityReports(v vulnerabilityScannerResponse, severityTypeSet map[string]bool) ([]matchResponse, error) {
var mr []matchResponse
mv := make([]vulnerabilityReport, 0)
if err := json.Unmarshal(v.Matches, &mr); err != nil {
return mv, fmt.Errorf("failed to unmarshal matches: %v", err)
return make([]matchResponse, 0), fmt.Errorf("failed to unmarshal matches: %v", err)
}
return filterVulnerabilityReportMatches(mr, severityTypeSet)
}

// parseVulerabilitiesReport unmarshals the vulnerability report and returns a list of vulnerabilities
// based on the severity levels set in severityTypeSet
func parseVulerabilitiesReport(filePath string, severityLevels []string) ([]vulnerabilityReport, error) {
mv := make([]vulnerabilityReport, 0)
func parseVulerabilitiesReport(filePath string, severityLevels []string) ([]matchResponse, error) {
mr := make([]matchResponse, 0)
data, err := os.ReadFile(filePath)
if err != nil {
return mv, fmt.Errorf("failed to read file at path %s: %v", filePath, err)
return mr, fmt.Errorf("failed to read file at path %s: %v", filePath, err)
}
var response vulnerabilityScannerResponse

if err = json.Unmarshal(data, &response); err != nil {
return mv, fmt.Errorf("failed to unmarshal response: %v", err)
return mr, fmt.Errorf("failed to unmarshal response: %v", err)
}
severityTypeSet := make(map[string]bool)
for _, severityLevel := range severityLevels {
Expand All @@ -75,13 +85,30 @@ func parseVulerabilitiesReport(filePath string, severityLevels []string) ([]vuln
}

// printResult Displays the filtered list of vulnerability reports to stdout
func printResult(mv []vulnerabilityReport) {
for _, vulnerability := range mv {
fmt.Printf("ID: %s\n", vulnerability.ID)
fmt.Printf("Severity: %s\n", vulnerability.Severity)
fmt.Printf("Namespace: %s\n", vulnerability.Namespace)
fmt.Printf("Description: %s\n", vulnerability.Description)
fmt.Printf("Fix Versions: %v\n", vulnerability.FixVersions)
func printResult(mr []matchResponse, githubActionOutput bool) {
for _, response := range mr {
fmt.Printf("ID: %s\n", response.Vulnerabilities.ID)
fmt.Printf("Link: https://github.com/advisories/%s\n", response.Vulnerabilities.DataSource)
fmt.Printf("Severity: %s\n", response.Vulnerabilities.Severity)
fmt.Printf("Namespace: %s\n", response.Vulnerabilities.Namespace)
fmt.Printf("Description: %s\n", response.Vulnerabilities.Description)
fmt.Printf("Fix Versions: %v\n", response.Vulnerabilities.FixVersions)
fmt.Println("Package:")
fmt.Printf("Name: %v\n", response.Artifact.Name)
fmt.Printf("Version: %v\n", response.Artifact.Version)
fmt.Printf("Type: %v\n", response.Artifact.Type)
fmt.Printf("PURL: %v\n", response.Artifact.Purl)
if githubActionOutput {
fmt.Println("::group::Locations")
fmt.Printf("%s\n", response.Artifact.Locations)
fmt.Println("::endgroup::")
fmt.Println("::group::Metadata")
fmt.Printf("%s\n", response.Artifact.Metadata)
fmt.Println("::endgroup::")
} else {
fmt.Printf("Locations: %s\n", response.Artifact.Locations)
fmt.Printf("Metadata: \n%s\n", response.Artifact.Metadata)
}
fmt.Printf("\n")
}
}
Expand All @@ -90,6 +117,7 @@ func main() {
validSeverityLevels := []string{"Negliable", "Low", "Medium", "High", "Critical"}
severityInputList := flag.String("s", "High,Critical", "Comma separated list of severity levels to scan. Valid severity levels are: "+strings.Join(validSeverityLevels, ","))
reportJsonFilePath := flag.String("p", "", "Path to the JSON file containing the vulnerabilities report")
githubActionOutput := flag.Bool("github", false, "Whether to use github action output format")
flag.Parse()

// passing file path is compulsory
Expand All @@ -98,15 +126,15 @@ func main() {
os.Exit(1)
}
severityLevels := strings.Split(*severityInputList, ",")
mv, err := parseVulerabilitiesReport(*reportJsonFilePath, severityLevels)
mr, err := parseVulerabilitiesReport(*reportJsonFilePath, severityLevels)
if err != nil {
fmt.Printf("Failed to parse vulnerabilities report: %v\n", err)
os.Exit(1)
}
fmt.Printf("Found %d vulnerabilities\n", len(mv))
if len(mv) == 0 {
fmt.Printf("Found %d vulnerabilities\n", len(mr))
if len(mr) == 0 {
os.Exit(0)
}
printResult(mv)
printResult(mr, *githubActionOutput)
os.Exit(1)
}
4 changes: 2 additions & 2 deletions pkg/tools/grype_report_parser_tool_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ func (v *VulnerabilityParserSuite) TestValidJsonForMatchingVulerabilities(c *C)
c.Assert(len(matchingVulnerabilities), Equals, 2)
c.Assert(err, IsNil)
for index, vulnerability := range matchingVulnerabilities {
c.Assert(vulnerability.ID, Equals, expectedIds[index])
c.Assert(vulnerability.Severity, Equals, severityLevels[index])
c.Assert(vulnerability.Vulnerabilities.ID, Equals, expectedIds[index])
c.Assert(vulnerability.Vulnerabilities.Severity, Equals, severityLevels[index])
}
}

0 comments on commit d2f3c33

Please sign in to comment.