feat: pluggable scanners

.families.yaml

@@ -121,3 +121,23 @@ rootkits:
   scanners_config:
     chkrootkit:
       binary_path: chkrootkit
+
+# TODO: decide how to use them across families, probably
+# referencing them from here would be the simplest way.
+plugins:
+  - name: binary-plugin
+    type: binary
+    path: path-to-binary
+    # config: some config data
+    # Note: communicates via CLI
+  - name: container-plugin
+    type: container
+    # version: v1
+    path: image-repo
+    # config: some config data, e.g. bootstrapping options, additional volumes, etc
+    # Note: communicates via embedded HTTP server via socket
+  - name: rest-plugin
+    type: rest-server
+    path: http://localhost:1234
+    # config: something to figure out?
+    # Note: communicates with the HTTP server directly

Makefile

@@ -293,7 +293,7 @@ docker-cr-discovery-server: ## Build K8S Image Resolver Docker image
 gen: gen-api gen-bicep gen-helm-docs ## Generating all code, manifests, docs
 
 .PHONY: gen-api
-gen-api: gen-apiserver-api gen-uibackend-api ## Generating API code
+gen-api: gen-apiserver-api gen-uibackend-api gen-scanner-api ## Generating API code
 
 .PHONY: gen-apiserver-api
 gen-apiserver-api: ## Generating Go library for API specification
@@ -309,6 +309,13 @@ gen-uibackend-api: ## Generating Go library for UI Backend API specification
 	go -C $(ROOT_DIR)/uibackend/client generate
 	go -C $(ROOT_DIR)/uibackend/server generate
 
+.PHONY: gen-scanner-api
+gen-scanner-api: ## Generating Go library for Scanner API specification
+	$(info Generating API for UI backend code ...)
+	go -C $(ROOT_DIR)/scanner/types generate
+	go -C $(ROOT_DIR)/scanner/client generate
+	go -C $(ROOT_DIR)/scanner/server generate
+
 .PHONY: gen-bicep
 gen-bicep: bin/bicep ## Generating Azure Bicep template(s)
 	$(info Generating Azure Bicep template(s) ...)

scanner/README.md

@@ -0,0 +1,68 @@
+## Pluggable scanners
+
+
+### Goals
+- standalone
+- quick to start
+- language-agnostic via client stubs for other languages
+- modular
+- scalable, albeit as a single service for now (no time to implement/rely on queues)
+- can be run as a container (kuberentes job or REST service), CLI command, or binary
+- can perform long running operations
+- replace current golang only approach for scanning
+- ...more things to capture here
+
+### Non-goals
+- durability, scans live only for the container lifetime
+  - SQLite DB can be exported as a file and reused, but did not try to make it work and probably will not to.
+  It will complicate more than benefit if done.
+- Replace centralized VMClarity DB for scanning results
+- ...more things to capture here
+
+
+## Brief overview
+Golang REST server implementation will be running across all scanner types (Go, Python, Rust, etc).
+Language-specific approaches will utilize the Go REST server as temporary DB and source of truth for scans (long running operations)
+and their results (scan findings).
+Other languages will only implement the scanner interface and interact with the REST server via client stubs (autogenerated with minimal changes) 
+that will be embedded into containers (MIGHT CHANGE).
+Use Golang scanner example to see how the new scanners will look like.
+
+## PoC
+
+### Running server
+
+```bash
+$ cd scanner/server                       # from repo root
+$ export TARGETPLATFORM=linux/arm64       # whatever works for you
+$ docker build . -t scanner-server        # build
+$ docker run -p 8765:8765 scanner-server  # start server
+```
+
+### Running example Golang scanner
+```bash
+$ cd scanner/examples/golang              # from repo root
+$ export TARGETPLATFORM=linux/arm64       # whatever works for you
+$ docker build . -t golang-scanner        # build
+
+## Pass env variables for scanner-server
+$ docker run -p 8765:8765 golang-scanner  # start scanner
+```
+
+### Running example Python scanner
+```bash
+$ cd scanner/examples/python              # from repo root
+$ export TARGETPLATFORM=linux/arm64       # whatever works for you
+$ docker build . -t python-scanner        # build
+
+## Pass env variables for scanner-server
+$ docker run -p 8765:8765 python-scanner  # start scanner
+```
+
+
+
+#### TODO
+- Fix API query and DB builders (metadata selector has some transient bugs due to ORM)
+- Add Docker images to Makefile
+- Resolve leftover package setup (to conform to the existing specifications)
+- Add simple E2E docker compose framework same as implemented for other components

scanner/client/client.cfg.yaml

@@ -0,0 +1,10 @@
+---
+package: client
+output: internal/client/client.gen.go
+additional-imports:
+  - package: github.com/openclarity/vmclarity/scanner/types
+    alias: .
+generate:
+  client: true
+output-options:
+  skip-prune: true

scanner/client/client.go

@@ -0,0 +1,81 @@
+package client
+
+import (
+	"context"
+	"encoding/json"
+	internal "github.com/openclarity/vmclarity/scanner/client/internal/client"
+	"github.com/openclarity/vmclarity/scanner/types"
+	"io"
+	"net/http"
+)
+
+// RequestEditorFn  is the function signature for the RequestEditor callback function
+type RequestEditorFn = internal.RequestEditorFn
+
+type Client struct {
+	api *internal.Client
+}
+
+func NewClient(server string) (*Client, error) {
+	c, err := internal.NewClient(server)
+	if err != nil {
+		return nil, err
+	}
+	return &Client{c}, nil
+}
+
+func (c *Client) GetFindings(ctx context.Context, params *types.GetFindingsParams, reqEditors ...RequestEditorFn) (*types.ScanFindings, error) {
+	return respTo[*types.ScanFindings](c.api.GetFindings(ctx, params, reqEditors...))
+}
+
+func (c *Client) GetScanFindingsForScan(ctx context.Context, scanID string, reqEditors ...RequestEditorFn) (*types.ScanFindings, error) {
+	return respTo[*types.ScanFindings](c.api.GetScanFindingsForScan(ctx, scanID, reqEditors...))
+}
+
+func (c *Client) IsAlive(ctx context.Context, reqEditors ...RequestEditorFn) (string, error) {
+	return respTo[string](c.api.IsAlive(ctx, reqEditors...))
+}
+
+func (c *Client) IsReady(ctx context.Context, reqEditors ...RequestEditorFn) (string, error) {
+	return respTo[string](c.api.IsReady(ctx, reqEditors...))
+}
+
+func (c *Client) GetScan(ctx context.Context, scanID string, reqEditors ...RequestEditorFn) (*types.Scan, error) {
+	return respTo[*types.Scan](c.api.GetScan(ctx, scanID, reqEditors...))
+}
+
+func (c *Client) SubmitScanEvent(ctx context.Context, scanID string, event types.ScanEvent, reqEditors ...RequestEditorFn) (*types.Scan, error) {
+	return respTo[*types.Scan](c.api.SubmitScanEvent(ctx, scanID, types.SubmitScanEventJSONRequestBody{
+		EventInfo: event.EventInfo,
+	}, reqEditors...))
+}
+
+func (c *Client) MarkScanAborted(ctx context.Context, scanID string, reqEditors ...RequestEditorFn) (*types.Scan, error) {
+	return respTo[*types.Scan](c.api.MarkScanAborted(ctx, scanID, reqEditors...))
+}
+
+func (c *Client) GetScans(ctx context.Context, params types.GetScansParams, reqEditors ...RequestEditorFn) (*types.Scans, error) {
+	return respTo[*types.Scans](c.api.GetScans(ctx, &params, reqEditors...))
+}
+
+func (c *Client) CreateScan(ctx context.Context, scan types.Scan, body io.Reader, reqEditors ...RequestEditorFn) (*types.Scan, error) {
+	return respTo[*types.Scan](c.api.CreateScan(ctx, scan, reqEditors...))
+}
+
+func respTo[T any](r *http.Response, err error) (T, error) {
+	var target T
+
+	// check original error
+	if err != nil {
+		return target, err
+	}
+
+	// decode
+	defer r.Body.Close()
+	err = json.NewDecoder(r.Body).Decode(&target)
+	if err != nil {
+		return target, err
+	}
+
+	return target, nil
+}

scanner/client/gen.go

@@ -0,0 +1,18 @@
+// Copyright © 2024 Cisco Systems, Inc. and its affiliates.
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package client
+
+//go:generate go run github.com/deepmap/oapi-codegen/v2/cmd/oapi-codegen --config=client.cfg.yaml ../openapi.yaml

scanner/client/go.mod

@@ -0,0 +1,29 @@
+module github.com/openclarity/vmclarity/scanner/client
+
+go 1.21.4
+
+require (
+	github.com/deepmap/oapi-codegen/v2 v2.1.0
+	github.com/oapi-codegen/runtime v1.1.1
+	github.com/openclarity/vmclarity/scanner/types v0.0.0-00010101000000-000000000000
+)
+
+require (
+	github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
+	github.com/getkin/kin-openapi v0.123.0 // indirect
+	github.com/go-openapi/jsonpointer v0.20.2 // indirect
+	github.com/go-openapi/swag v0.22.8 // indirect
+	github.com/google/uuid v1.5.0 // indirect
+	github.com/invopop/yaml v0.2.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
+	github.com/perimeterx/marshmallow v1.1.5 // indirect
+	golang.org/x/mod v0.15.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/tools v0.18.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)
+
+replace github.com/openclarity/vmclarity/scanner/types => ../types

scanner/client/go.sum

@@ -0,0 +1,64 @@
+github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk=
+github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ=
+github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk=
+github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/deepmap/oapi-codegen/v2 v2.1.0 h1:I/NMVhJCtuvL9x+S2QzZKpSjGi33oDZwPRdemvOZWyQ=
+github.com/deepmap/oapi-codegen/v2 v2.1.0/go.mod h1:R1wL226vc5VmCNJUvMyYr3hJMm5reyv25j952zAVXZ8=
+github.com/getkin/kin-openapi v0.123.0 h1:zIik0mRwFNLyvtXK274Q6ut+dPh6nlxBp0x7mNrPhs8=
+github.com/getkin/kin-openapi v0.123.0/go.mod h1:wb1aSZA/iWmorQP9KTAS/phLj/t17B5jT7+fS8ed9NM=
+github.com/go-openapi/jsonpointer v0.20.2 h1:mQc3nmndL8ZBzStEo3JYF8wzmeWffDH4VbXz58sAx6Q=
+github.com/go-openapi/jsonpointer v0.20.2/go.mod h1:bHen+N0u1KEO3YlmqOjTT9Adn1RfD91Ar825/PuiRVs=
+github.com/go-openapi/swag v0.22.8 h1:/9RjDSQ0vbFR+NyjGMkFTsA1IA0fmhKSThmfGZjicbw=
+github.com/go-openapi/swag v0.22.8/go.mod h1:6QT22icPLEqAM/z/TChgb4WAveCHF92+2gF0CNjHpPI=
+github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM=
+github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
+github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU=
+github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/invopop/yaml v0.2.0 h1:7zky/qH+O0DwAyoobXUqvVBwgBFRxKoQ/3FjcVpjTMY=
+github.com/invopop/yaml v0.2.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
+github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
+github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmtpMYro=
+github.com/oapi-codegen/runtime v1.1.1/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg=
+github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX50IvK2s=
+github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
+github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
+github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU=
+github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
+golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug=
+golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mod v0.15.0 h1:SernR4v+D55NyBH2QiEQrlBAnj1ECL6AGrA5+dPaMY8=
+golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ=
+golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=