From 2494e88bf90b7265e22f1f96903be32df16c7a45 Mon Sep 17 00:00:00 2001 From: Erez Fishhimer Date: Thu, 10 Nov 2022 03:52:03 +0200 Subject: [PATCH 01/23] discovery + run job --- .gitignore | 2 +- runtime_scan/go.mod | 76 +++ runtime_scan/go.sum | 613 ++++++++++++++++++ runtime_scan/pkg/config/config.go | 60 ++ runtime_scan/pkg/config/delete_job_policy.go | 33 + runtime_scan/pkg/config/scan_config.go | 68 ++ runtime_scan/pkg/orchestrator/orchestrator.go | 109 ++++ runtime_scan/pkg/provider/aws/client.go | 339 ++++++++++ runtime_scan/pkg/provider/aws/client_test.go | 55 ++ runtime_scan/pkg/provider/provider.go | 24 + runtime_scan/pkg/scanner/common.go | 27 + runtime_scan/pkg/scanner/job_managment.go | 374 +++++++++++ runtime_scan/pkg/scanner/scanner.go | 206 ++++++ runtime_scan/pkg/types/errors.go | 35 + runtime_scan/pkg/types/types.go | 85 +++ 15 files changed, 2105 insertions(+), 1 deletion(-) create mode 100644 runtime_scan/go.mod create mode 100644 runtime_scan/go.sum create mode 100644 runtime_scan/pkg/config/config.go create mode 100644 runtime_scan/pkg/config/delete_job_policy.go create mode 100644 runtime_scan/pkg/config/scan_config.go create mode 100644 runtime_scan/pkg/orchestrator/orchestrator.go create mode 100644 runtime_scan/pkg/provider/aws/client.go create mode 100644 runtime_scan/pkg/provider/aws/client_test.go create mode 100644 runtime_scan/pkg/provider/provider.go create mode 100644 runtime_scan/pkg/scanner/common.go create mode 100644 runtime_scan/pkg/scanner/job_managment.go create mode 100644 runtime_scan/pkg/scanner/scanner.go create mode 100644 runtime_scan/pkg/types/errors.go create mode 100644 runtime_scan/pkg/types/types.go diff --git a/.gitignore b/.gitignore index 92e7ee76e..ab290bfcd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ .DS_Store .idea/ -bin/ \ No newline at end of file +bin/ diff --git a/runtime_scan/go.mod b/runtime_scan/go.mod new file mode 100644 index 000000000..6da0cd59f --- /dev/null +++ b/runtime_scan/go.mod @@ -0,0 +1,76 @@ +module github.com/openclarity/vmclarity/runtime_scan + +go 1.19 + +require ( + github.com/aws/aws-sdk-go-v2/config v1.17.10 + github.com/aws/aws-sdk-go-v2/service/ec2 v1.66.0 + github.com/satori/go.uuid v1.2.0 + github.com/sirupsen/logrus v1.9.0 + github.com/spf13/viper v1.14.0 + k8s.io/client-go v0.25.3 +) + +require ( + github.com/PuerkitoBio/purell v1.1.1 // indirect + github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect + github.com/aws/aws-sdk-go-v2 v1.17.1 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.12.23 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.11.25 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.17.1 // indirect + github.com/aws/smithy-go v1.13.4 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/emicklei/go-restful/v3 v3.8.0 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/go-logr/logr v1.2.3 // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/jsonreference v0.19.5 // indirect + github.com/go-openapi/swag v0.19.14 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/google/gnostic v0.5.7-v3refs // indirect + github.com/google/gofuzz v1.1.0 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/magiconair/properties v1.8.6 // indirect + github.com/mailru/easyjson v0.7.6 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pelletier/go-toml v1.9.5 // indirect + github.com/pelletier/go-toml/v2 v2.0.5 // indirect + github.com/spf13/afero v1.9.2 // indirect + github.com/spf13/cast v1.5.0 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/subosito/gotenv v1.4.1 // indirect + golang.org/x/net v0.0.0-20221014081412-f15817d10f9b // indirect + golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect + golang.org/x/sys v0.0.0-20220908164124-27713097b956 // indirect + golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect + golang.org/x/text v0.4.0 // indirect + golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.28.1 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/api v0.25.3 // indirect + k8s.io/apimachinery v0.25.3 // indirect + k8s.io/klog/v2 v2.70.1 // indirect + k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect + k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed // indirect + sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect + sigs.k8s.io/yaml v1.2.0 // indirect +) diff --git a/runtime_scan/go.sum b/runtime_scan/go.sum new file mode 100644 index 000000000..eeb250779 --- /dev/null +++ b/runtime_scan/go.sum @@ -0,0 +1,613 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/aws/aws-sdk-go-v2 v1.17.1 h1:02c72fDJr87N8RAC2s3Qu0YuvMRZKNZJ9F+lAehCazk= +github.com/aws/aws-sdk-go-v2 v1.17.1/go.mod h1:JLnGeGONAyi2lWXI1p0PCIOIy333JMVK1U7Hf0aRFLw= +github.com/aws/aws-sdk-go-v2/config v1.17.10 h1:zBy5QQ/mkvHElM1rygHPAzuH+sl8nsdSaxSWj0+rpdE= +github.com/aws/aws-sdk-go-v2/config v1.17.10/go.mod h1:/4np+UiJJKpWHN7Q+LZvqXYgyjgeXm5+lLfDI6TPZao= +github.com/aws/aws-sdk-go-v2/credentials v1.12.23 h1:LctvcJMIb8pxvk5hQhChpCu0WlU6oKQmcYb1HA4IZSA= +github.com/aws/aws-sdk-go-v2/credentials v1.12.23/go.mod h1:0awX9iRr/+UO7OwRQFpV1hNtXxOVuehpjVEzrIAYNcA= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19 h1:E3PXZSI3F2bzyj6XxUXdTIfvp425HHhwKsFvmzBwHgs= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19/go.mod h1:VihW95zQpeKQWVPGkwT+2+WJNQV8UXFfMTWdU6VErL8= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25 h1:nBO/RFxeq/IS5G9Of+ZrgucRciie2qpLy++3UGZ+q2E= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25/go.mod h1:Zb29PYkf42vVYQY6pvSyJCJcFHlPIiY+YKdPtwnvMkY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19 h1:oRHDrwCTVT8ZXi4sr9Ld+EXk7N/KGssOr2ygNeojEhw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19/go.mod h1:6Q0546uHDp421okhmmGfbxzq2hBqbXFNpi4k+Q1JnQA= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26 h1:Mza+vlnZr+fPKFKRq/lKGVvM6B/8ZZmNdEopOwSQLms= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26/go.mod h1:Y2OJ+P+MC1u1VKnavT+PshiEuGPyh/7DqxoDNij4/bg= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.66.0 h1:yankZN/p8rKWHCgbj6N2SGeZ66XFqOS3Ud80DahavQs= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.66.0/go.mod h1:zul71QqzR4D1a90/5FloZiAnZ1CtuIjVH7R9MP997+A= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19 h1:GE25AWCdNUPh9AOJzI9KIJnja7IwUc1WyUqz/JTyJ/I= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19/go.mod h1:02CP6iuYP+IVnBX5HULVdSAku/85eHB2Y9EsFhrkEwU= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.25 h1:GFZitO48N/7EsFDt8fMa5iYdmWqkUDDB3Eje6z3kbG0= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.25/go.mod h1:IARHuzTXmj1C0KS35vboR0FeJ89OkEy1M9mWbK2ifCI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 h1:jcw6kKZrtNfBPJkaHrscDOZoe5gvi9wjudnxvozYFJo= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8/go.mod h1:er2JHN+kBY6FcMfcBBKNGCT3CarImmdFzishsqBmSRI= +github.com/aws/aws-sdk-go-v2/service/sts v1.17.1 h1:KRAix/KHvjGODaHAMXnxRk9t0D+4IJVUuS/uwXxngXk= +github.com/aws/aws-sdk-go-v2/service/sts v1.17.1/go.mod h1:bXcN3koeVYiJcdDU89n3kCYILob7Y34AeLopUbZgLT4= +github.com/aws/smithy-go v1.13.4 h1:/RN2z1txIJWeXeOkzX+Hk/4Uuvv7dWtCjbmVJcrskyk= +github.com/aws/smithy-go v1.13.4/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/emicklei/go-restful/v3 v3.8.0 h1:eCZ8ulSerjdAiaNpF7GxXIE7ZCMo1moN1qX+S609eVw= +github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.19.5 h1:1WJP/wi4OjB4iV8KVbH73rQaoialJrqv8gitZLxGLtM= +github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= +github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= +github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= +github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/onsi/ginkgo/v2 v2.1.6 h1:Fx2POJZfKRQcM1pH49qSZiYeu319wji004qX+GDovrU= +github.com/onsi/gomega v1.20.1 h1:PA/3qinGoukvymdIDV8pii6tiZgC8kbmJO6Z5+b002Q= +github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= +github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml/v2 v2.0.5 h1:ipoSadvV8oGUjnUbMub59IDPPwfxF694nG/jwbMiyQg= +github.com/pelletier/go-toml/v2 v2.0.5/go.mod h1:OMHamSCAODeSsVrwwvcJOaoN0LIUIaFVNZzmWyNfXas= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= +github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.9.2 h1:j49Hj62F0n+DaZ1dDCvhABaPNSGNkt32oRFxI33IEMw= +github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= +github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= +github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.14.0 h1:Rg7d3Lo706X9tHsJMUjdiwMpHB7W8WnSVOssIY+JElU= +github.com/spf13/viper v1.14.0/go.mod h1:WT//axPky3FdvXHzGw33dNdXXXfFQqmEalje+egj8As= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= +github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20221014081412-f15817d10f9b h1:tvrvnPFcdzp294diPnrdZZZ8XUt2Tyj7svb7X52iDuU= +golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 h1:nt+Q6cXKz4MosCSpnbMtqiQ8Oz0pxTef2B4Vca2lvfk= +golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956 h1:XeJjHH1KiLpKGb6lvMiksZ9l0fVUh+AmGcm0nOMEBOY= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220609170525-579cf78fd858 h1:Dpdu/EMxGMFgq0CeYMh4fazTD2vtlZRYE7wyynxJb9U= +golang.org/x/time v0.0.0-20220609170525-579cf78fd858/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.25.3 h1:Q1v5UFfYe87vi5H7NU0p4RXC26PPMT8KOpr1TLQbCMQ= +k8s.io/api v0.25.3/go.mod h1:o42gKscFrEVjHdQnyRenACrMtbuJsVdP+WVjqejfzmI= +k8s.io/apimachinery v0.25.3 h1:7o9ium4uyUOM76t6aunP0nZuex7gDf8VGwkR5RcJnQc= +k8s.io/apimachinery v0.25.3/go.mod h1:jaF9C/iPNM1FuLl7Zuy5b9v+n35HGSh6AQ4HYRkCqwo= +k8s.io/client-go v0.25.3 h1:oB4Dyl8d6UbfDHD8Bv8evKylzs3BXzzufLiO27xuPs0= +k8s.io/client-go v0.25.3/go.mod h1:t39LPczAIMwycjcXkVc+CB+PZV69jQuNx4um5ORDjQA= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.70.1 h1:7aaoSdahviPmR+XkS7FyxlkkXs6tHISSG03RxleQAVQ= +k8s.io/klog/v2 v2.70.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 h1:MQ8BAZPZlWk3S9K4a9NCkIFQtZShWqoha7snGixVgEA= +k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1/go.mod h1:C/N6wCaBHeBHkHUesQOQy2/MZqGgMAFPqGsGQLdbZBU= +k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed h1:jAne/RjBTyawwAy0utX5eqigAwz/lQhTmy+Hr/Cpue4= +k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= +sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= +sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/runtime_scan/pkg/config/config.go b/runtime_scan/pkg/config/config.go new file mode 100644 index 000000000..5e5210513 --- /dev/null +++ b/runtime_scan/pkg/config/config.go @@ -0,0 +1,60 @@ +// Copyright © 2022 Cisco Systems, Inc. and its affiliates. +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "github.com/spf13/viper" + "k8s.io/client-go/kubernetes" +) + +const ( + ScannerJobResultListenPort = "SCANNER_JOB_RESULT_LISTEN_PORT" + CredsSecretNamespace = "CREDS_SECRET_NAMESPACE" // nolint: gosec + ScannerJobTemplateConfigMapName = "SCANNER_JOB_TEMPLATE_CONFIG_MAP_NAME" + ScannerJobTemplateConfigMapNamespace = "SCANNER_JOB_TEMPLATE_CONFIG_MAP_NAMESPACE" + defaultScannerJobResultListenPort = 8888 +) + +type Config struct { + ScannerJobResultListenPort int + CredsSecretNamespace string + Region string + VpcID string + SubnetID string + AmiID string +} + +func setConfigDefaults() { + viper.SetDefault(CredsSecretNamespace, "kubeclarity") + viper.SetDefault(ScannerJobTemplateConfigMapName, "") + viper.SetDefault(ScannerJobTemplateConfigMapNamespace, "kubeclarity") + viper.SetDefault(ScannerJobResultListenPort, defaultScannerJobResultListenPort) + + viper.AutomaticEnv() +} + +func LoadConfig(clientset kubernetes.Interface) (*Config, error) { + setConfigDefaults() + + + + config := &Config{ + ScannerJobResultListenPort: viper.GetInt(ScannerJobResultListenPort), + CredsSecretNamespace: viper.GetString(CredsSecretNamespace), + } + + return config, nil +} diff --git a/runtime_scan/pkg/config/delete_job_policy.go b/runtime_scan/pkg/config/delete_job_policy.go new file mode 100644 index 000000000..be497b860 --- /dev/null +++ b/runtime_scan/pkg/config/delete_job_policy.go @@ -0,0 +1,33 @@ +// Copyright © 2022 Cisco Systems, Inc. and its affiliates. +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +type DeleteJobPolicyType string + +const ( + DeleteJobPolicyAll DeleteJobPolicyType = "All" + DeleteJobPolicyNever DeleteJobPolicyType = "Never" + DeleteJobPolicySuccessful DeleteJobPolicyType = "Successful" +) + +func (dj DeleteJobPolicyType) IsValid() bool { + switch dj { + case DeleteJobPolicyAll, DeleteJobPolicyNever, DeleteJobPolicySuccessful: + return true + default: + return false + } +} diff --git a/runtime_scan/pkg/config/scan_config.go b/runtime_scan/pkg/config/scan_config.go new file mode 100644 index 000000000..a0c329e41 --- /dev/null +++ b/runtime_scan/pkg/config/scan_config.go @@ -0,0 +1,68 @@ +// Copyright © 2022 Cisco Systems, Inc. and its affiliates. +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "time" + + log "github.com/sirupsen/logrus" + "github.com/spf13/viper" + + "github.com/openclarity/vmclarity/runtime_scan/pkg/types" +) + +const ( + JobResultTimeout = "JOB_RESULT_TIMEOUT" + MaxParallelism = "MAX_PARALLELISM" + DeleteJobPolicy = "DELETE_JOB_POLICY" +) + +type ScanConfig struct { + MaxScanParallelism int + // instances to scan + Instances []types.Instance + DiscoveryFilters types.ScanScope + JobResultTimeout time.Duration + DeleteJobPolicy DeleteJobPolicyType +} + +func setScanConfigDefaults() { + viper.SetDefault(MaxParallelism, "10") + viper.SetDefault(JobResultTimeout, "10m") + viper.SetDefault(DeleteJobPolicy, DeleteJobPolicySuccessful) + + viper.AutomaticEnv() +} + +func LoadScanConfig() *ScanConfig { + setScanConfigDefaults() + + return &ScanConfig{ + MaxScanParallelism: viper.GetInt(MaxParallelism), + JobResultTimeout: viper.GetDuration(JobResultTimeout), + DeleteJobPolicy: getDeleteJobPolicyType(viper.GetString(DeleteJobPolicy)), + } +} + +func getDeleteJobPolicyType(policyType string) DeleteJobPolicyType { + deleteJobPolicy := DeleteJobPolicyType(policyType) + if !deleteJobPolicy.IsValid() { + log.Warnf("Invalid %s type - using default `%s`", DeleteJobPolicy, DeleteJobPolicySuccessful) + deleteJobPolicy = DeleteJobPolicySuccessful + } + + return deleteJobPolicy +} diff --git a/runtime_scan/pkg/orchestrator/orchestrator.go b/runtime_scan/pkg/orchestrator/orchestrator.go new file mode 100644 index 000000000..d62d2646e --- /dev/null +++ b/runtime_scan/pkg/orchestrator/orchestrator.go @@ -0,0 +1,109 @@ +// Copyright © 2022 Cisco Systems, Inc. and its affiliates. +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package orchestrator + +import ( + "fmt" + "sync" + + log "github.com/sirupsen/logrus" + + _config "github.com/openclarity/vmclarity/runtime_scan/pkg/config" + "github.com/openclarity/vmclarity/runtime_scan/pkg/provider" + aws2 "github.com/openclarity/vmclarity/runtime_scan/pkg/provider/aws" + _scanner "github.com/openclarity/vmclarity/runtime_scan/pkg/scanner" +) + +type Orchestrator struct { + scanner *_scanner.Scanner + config *_config.Config + providerClient provider.Client + sync.Mutex +} + +//go:generate $GOPATH/bin/mockgen -destination=./mock_orchestrator.go -package=orchestrator github.com/openclarity/kubeclarity/runtime_scan/pkg/orchestrator VulnerabilitiesScanner +type VulnerabilitiesScanner interface { + Start(errChan chan struct{}) + Scan(scanConfig *_config.ScanConfig, scanDone chan struct{}) error + //ScanProgress() types.ScanProgress + ///Results() *types.ScanResults + //Clear() + Stop() +} + +func Create(config *_config.Config) (*Orchestrator, error) { + // for now will statically create aws client here (until we support more cloud providers) + awsClient, err := aws2.Create() + if err != nil { + return nil, fmt.Errorf("failed to create aws client: %v", err) + } + + orc := &Orchestrator{ + config: config, + providerClient: awsClient, + Mutex: sync.Mutex{}, + } + + return orc, nil +} + +func (o *Orchestrator) Start(errChan chan struct{}) { + // Start result server + log.Infof("Starting Orchestrator server") +} + +func (o *Orchestrator) Stop() { + log.Infof("Stopping Orchestrator server") +} + +func (o *Orchestrator) Scan(scanConfig *_config.ScanConfig, scanDone chan struct{}) error { + instances, err := o.providerClient.Discover(&scanConfig.DiscoveryFilters) + if err != nil { + return err + } + scanConfig.Instances = instances + + if err := o.getScanner().Scan(scanConfig, scanDone); err != nil { + return fmt.Errorf("failed to scan: %v", err) + } + + return nil +} + +// +//func (o *Orchestrator) ScanProgress() types.ScanProgress { +// return o.getScanner().ScanProgress() +//} +// +//func (o *Orchestrator) Results() *types.ScanResults { +// return o.getScanner().Results() +//} + +//func (o *Orchestrator) Clear() { +// o.Lock() +// defer o.Unlock() +// +// log.Infof("Clearing Orchestrator") +// o.scanner.Clear() +// //o.scanner = _scanner.CreateScanner(o.config, o.clientset) +//} + +func (o *Orchestrator) getScanner() *_scanner.Scanner { + o.Lock() + defer o.Unlock() + + return o.scanner +} diff --git a/runtime_scan/pkg/provider/aws/client.go b/runtime_scan/pkg/provider/aws/client.go new file mode 100644 index 000000000..f0b04aad0 --- /dev/null +++ b/runtime_scan/pkg/provider/aws/client.go @@ -0,0 +1,339 @@ +package aws + +import ( + "context" + "fmt" + "strings" + "time" + + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + + "github.com/openclarity/vmclarity/runtime_scan/pkg/types" +) + +var ( + snapshotDescription = "VMClarity snapshot" + tagKey = "Owner" + tagVal = "VMClarity" + vmclarityTags = []ec2types.Tag{ + { + Key: &tagKey, + Value: &tagVal, + }, + } +) + +type Client struct { + ec2Client *ec2.Client +} + +func Create() (*Client, error) { + var awsClient Client + + cfg, err := awsconfig.LoadDefaultConfig(context.TODO()) + if err != nil { + return nil, fmt.Errorf("failed to load aws config: %v", err) + } + + awsClient.ec2Client = ec2.NewFromConfig(cfg) + + return &awsClient, nil +} + +func (c *Client) Discover(scope *types.ScanScope) ([]types.Instance, error) { + var ret []types.Instance + + regions, err := c.getRegionsToScan(scope) + if err != nil { + return nil, fmt.Errorf("failed to get regions to scan: %v", err) + } + if len(regions) == 0 { + return nil, fmt.Errorf("no regions to scan") + } + + for _, region := range regions { + describeFilters := createDescribeFilters(scope, region) + + out, err := c.ec2Client.DescribeInstances(context.TODO(), &ec2.DescribeInstancesInput{ + Filters: describeFilters, + MaxResults: nil, // TODO + NextToken: nil, // TODO + }, func(options *ec2.Options) { + options.Region = region + }) + if err != nil { + return nil, fmt.Errorf("failed to describe instances: %v", err) + } + // get all returned instances: + for _, reservation := range out.Reservations { + for _, instance := range reservation.Instances { + if instance.InstanceId != nil { + ret = append(ret, types.Instance{ + ID: *instance.InstanceId, + Region: region, + }) + } + } + } + } + return ret, nil +} + +func (c *Client) CreateSnapshot(volume types.Volume) (types.Snapshot, error) { + params := ec2.CreateSnapshotInput{ + VolumeId: &volume.ID, + Description: &snapshotDescription, + TagSpecifications: []ec2types.TagSpecification{ + { + ResourceType: ec2types.ResourceTypeSnapshot, + Tags: vmclarityTags, + }, + }, + } + out, err := c.ec2Client.CreateSnapshot(context.TODO(), ¶ms, func(options *ec2.Options) { + options.Region = volume.Region + }) + if err != nil { + return types.Snapshot{}, fmt.Errorf("failed to create snapshot: %v", err) + } + return types.Snapshot{ + ID: *out.SnapshotId, + Region: volume.Region, + }, nil +} + +func (c *Client) WaitForSnapshotReady(snapshot types.Snapshot) error { + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + timeout := time.After(2 * time.Minute) + + for { + select { + case <-ticker.C: + out, err := c.ec2Client.DescribeSnapshots(context.TODO(), &ec2.DescribeSnapshotsInput{ + SnapshotIds: []string{snapshot.ID}, + }, func(options *ec2.Options) { + options.Region = snapshot.Region + }) + if err != nil { + return fmt.Errorf("failed to descrive snapshot: %v", err) + } + if out.Snapshots[0].State == ec2types.SnapshotStateCompleted { + return nil + } + case <-timeout: + return fmt.Errorf("timeout") + } + } +} + +func (c *Client) CopySnapshot(snapshot types.Snapshot, dstRegion string) (types.Snapshot, error) { + snap, err := c.ec2Client.CopySnapshot(context.TODO(), &ec2.CopySnapshotInput{ + SourceRegion: &snapshot.Region, + SourceSnapshotId: &snapshot.ID, + Description: &snapshotDescription, + TagSpecifications: []ec2types.TagSpecification{ + { + ResourceType: ec2types.ResourceTypeSnapshot, + Tags: vmclarityTags, + }, + }, + }, func(options *ec2.Options) { + options.Region = dstRegion + }) + if err != nil { + return types.Snapshot{}, fmt.Errorf("failed to copy snapshot: %v", err) + } + + return types.Snapshot{ + ID: *snap.SnapshotId, + Region: dstRegion, + }, nil +} + +func (c *Client) GetInstanceRootVolume(instance types.Instance) (types.Volume, error) { + out, err := c.ec2Client.DescribeInstances(context.TODO(), &ec2.DescribeInstancesInput{ + InstanceIds: []string{instance.ID}, + }, func(options *ec2.Options) { + options.Region = instance.Region + }) + if err != nil { + return types.Volume{}, fmt.Errorf("failed to describe instances: %v", err) + } + + if len(out.Reservations) == 0 { + return types.Volume{}, fmt.Errorf("no reservations were found") + } + if len(out.Reservations) > 1 { + return types.Volume{}, fmt.Errorf("more than one reservations were found") + } + if len(out.Reservations[0].Instances) == 0 { + return types.Volume{}, fmt.Errorf("no instances were found") + } + if len(out.Reservations[0].Instances) > 1 { + return types.Volume{}, fmt.Errorf("more than one instances were found") + } + + outInstance := out.Reservations[0].Instances[0] + rootDeviceName := *outInstance.RootDeviceName + + for _, blkDevice := range outInstance.BlockDeviceMappings { + if strings.Compare(*blkDevice.DeviceName, rootDeviceName) == 0 { + return types.Volume{ + ID: *blkDevice.Ebs.VolumeId, + Name: rootDeviceName, + Region: instance.Region, + }, nil + } + } + return types.Volume{}, fmt.Errorf("failed to find root device volume") +} + +func (c *Client) LaunchInstance(ami, deviceName string, snapshot types.Snapshot) (types.Instance, error) { + var max = int32(1) + var min = int32(1) + + out, err := c.ec2Client.RunInstances(context.TODO(), &ec2.RunInstancesInput{ + MaxCount: &max, + MinCount: &min, + ImageId: &ami, + BlockDeviceMappings: []ec2types.BlockDeviceMapping{ + { + DeviceName: &deviceName, + Ebs: &ec2types.EbsBlockDevice{ + DeleteOnTermination: nil, // ? + Encrypted: nil, // ? + SnapshotId: &snapshot.ID, + VolumeSize: nil, // default is snapshot size + VolumeType: ec2types.VolumeTypeGp2, // ? + }, + }, + }, + InstanceType: ec2types.InstanceTypeT2Large, + MetadataOptions: nil, // ? + SecurityGroups: nil, // use default for now + SubnetId: nil, // use default for now + TagSpecifications: nil, // need to specify tags + UserData: nil, // put launch script here + }, func(options *ec2.Options) { + options.Region = snapshot.Region + }) + if err != nil { + return types.Instance{}, fmt.Errorf("failed to run instances: %v", err) + } + + return types.Instance{ + ID: *out.Instances[0].InstanceId, + Region: snapshot.Region, + }, nil +} + +func (c *Client) DeleteInstance(instance types.Instance) error { + _, err := c.ec2Client.TerminateInstances(context.TODO(), &ec2.TerminateInstancesInput{ + InstanceIds: []string{instance.ID}, + }, func(options *ec2.Options) { + options.Region = instance.Region + }) + if err != nil { + return fmt.Errorf("failed to terminate instances: %v", err) + } + + return nil +} + +func (c *Client) DeleteSnapshot(snapshot types.Snapshot) error { + _, err := c.ec2Client.DeleteSnapshot(context.TODO(), &ec2.DeleteSnapshotInput{ + SnapshotId: &snapshot.ID, + }, func(options *ec2.Options) { + options.Region = snapshot.Region + }) + if err != nil { + return fmt.Errorf("failed to delete snapshot: %v", err) + } + + return nil +} + +func findRegionByID(regions []types.Region, regionID string) types.Region { + for _, region := range regions { + if strings.Compare(region.ID, regionID) == 0 { + return region + } + } + return types.Region{} +} + +func getRegionVPCsAndSecurityGroups(region types.Region) (vpcs []string, sgs []string) { + for _, vpc := range region.VPCs { + vpcs = append(vpcs, vpc.ID) + for _, sc := range vpc.SecurityGroups { + sgs = append(sgs, sc.ID) + } + } + return +} + +func createDescribeFilters(scopes *types.ScanScope, regionID string) []ec2types.Filter { + var ret []ec2types.Filter + var vpcs []string + var sgs []string + + if !scopes.All { + region := findRegionByID(scopes.Regions, regionID) + vpcs, sgs = getRegionVPCsAndSecurityGroups(region) + } + + vpcID := "vpc-id" + sgID := "instance.group-id" // TODO + + if len(sgs) > 0 { + ret = append(ret, ec2types.Filter{ + Name: &sgID, + Values: sgs, + }) + } else if len(vpcs) > 0 { + ret = append(ret, ec2types.Filter{ + Name: &vpcID, + Values: vpcs, + }) + } + + if len(scopes.IncludeTags) > 0 { + for _, tag := range scopes.IncludeTags { + name := "tag:" + tag.Key + ret = append(ret, ec2types.Filter{ + Name: &name, + Values: []string{tag.Val}, + }) + } + } + + return ret +} + +func (c *Client) getRegionsToScan(scope *types.ScanScope) ([]string, error) { + if scope.All { + return c.ListAllRegions() + } + + var ret []string + for _, region := range scope.Regions { + ret = append(ret, region.ID) + } + + return ret, nil +} + +func (c *Client) ListAllRegions() ([]string, error) { + var ret []string + out, err := c.ec2Client.DescribeRegions(context.TODO(), &ec2.DescribeRegionsInput{}) + if err != nil { + return nil, err + } + for _, region := range out.Regions { + ret = append(ret, *region.RegionName) + } + return ret, nil +} diff --git a/runtime_scan/pkg/provider/aws/client_test.go b/runtime_scan/pkg/provider/aws/client_test.go new file mode 100644 index 000000000..f36ebcd82 --- /dev/null +++ b/runtime_scan/pkg/provider/aws/client_test.go @@ -0,0 +1,55 @@ +package aws + +import ( + "context" + "testing" + + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/ec2" + + "github.com/openclarity/vmclarity/runtime_scan/pkg/types" +) + +func TestClient_ListAllRegions(t *testing.T) { + cfg, err := awsconfig.LoadDefaultConfig(context.TODO()) + if err != nil { + t.Fatalf("%v", err) + } + ec2Client := ec2.NewFromConfig(cfg) + + c := Client{ + ec2Client: ec2Client, + } + instance := types.Instance{ + ID: "i-0f70b335ea12b2853", + Region: "us-east-1", + } + rootVolume, err := c.GetInstanceRootVolume(instance) + if err != nil { + t.Fatalf("%v", err) + } + // create a snapshot of that vm + srcSnapshot, err := c.CreateSnapshot(rootVolume) + if err != nil { + t.Fatalf("%v", err) + } + if err := c.WaitForSnapshotReady(srcSnapshot); err != nil { + t.Fatalf("%v", err) + } + //copy the snapshot to the scanner region + // TODO make sure we need this. + cpySnapshot, err := c.CopySnapshot(srcSnapshot, "us-east-2") + if err != nil { + t.Fatalf("%v", err) + } + if err := c.WaitForSnapshotReady(cpySnapshot); err != nil { + t.Fatalf("%v", err) + } + // create the scanner job (vm) with a boot script + launchedInstance, err := c.LaunchInstance("ami-0568773882d492fc8", "xvdh", cpySnapshot) + if err != nil { + t.Fatalf("%v", err) + } + + t.Logf("res: %v", launchedInstance.ID) +} diff --git a/runtime_scan/pkg/provider/provider.go b/runtime_scan/pkg/provider/provider.go new file mode 100644 index 000000000..dbe64d5ed --- /dev/null +++ b/runtime_scan/pkg/provider/provider.go @@ -0,0 +1,24 @@ +package provider + +import types2 "github.com/openclarity/vmclarity/runtime_scan/pkg/types" + +type Client interface { + // list VM instance ids in the account according to the filters. + Discover(filters *types2.ScanScope) ([]types2.Instance, error) + // create a snapshot of a volume, and return the snapshot id. + CreateSnapshot(types2.Volume) (types2.Snapshot, error) + // copy the snapshot from src region to dest region and return the snapshot id. + CopySnapshot(snapshot types2.Snapshot, dstRegion string) (types2.Snapshot, error) + // + WaitForSnapshotReady(snapshot types2.Snapshot) error + // get the instance root volume + GetInstanceRootVolume(instance types2.Instance) (types2.Volume, error) + // attach a volume to an instance + //AttachVolume(volumeID, instanceID, region string) (string, error) + // + LaunchInstance(ami, deviceName string, snapshot types2.Snapshot) (types2.Instance, error) + // + DeleteInstance(types2.Instance) error + // + DeleteSnapshot(types2.Snapshot) error +} diff --git a/runtime_scan/pkg/scanner/common.go b/runtime_scan/pkg/scanner/common.go new file mode 100644 index 000000000..38d646750 --- /dev/null +++ b/runtime_scan/pkg/scanner/common.go @@ -0,0 +1,27 @@ +// Copyright © 2022 Cisco Systems, Inc. and its affiliates. +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scanner + +func nonBlockingNotification(scanDone chan struct{}) { + if scanDone == nil { + return + } + + select { + case scanDone <- struct{}{}: + default: + } +} diff --git a/runtime_scan/pkg/scanner/job_managment.go b/runtime_scan/pkg/scanner/job_managment.go new file mode 100644 index 000000000..fe4e5f082 --- /dev/null +++ b/runtime_scan/pkg/scanner/job_managment.go @@ -0,0 +1,374 @@ +// Copyright © 2022 Cisco Systems, Inc. and its affiliates. +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scanner + +import ( + "fmt" + "sync/atomic" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/openclarity/vmclarity/runtime_scan/pkg/config" + types2 "github.com/openclarity/vmclarity/runtime_scan/pkg/types" +) + +// run jobs. +func (s *Scanner) jobBatchManagement(scanDone chan struct{}) { + s.Lock() + imageIDToScanData := s.instanceIDToScanData + numberOfWorkers := s.scanConfig.MaxScanParallelism + imagesStartedToScan := &s.progress.ImagesStartedToScan + imagesCompletedToScan := &s.progress.ImagesCompletedToScan + s.Unlock() + + // queue of scan data + q := make(chan *scanData) + // done channel takes the result of the job + done := make(chan bool) + + fullScanDone := make(chan bool) + + // spawn workers + for i := 0; i < numberOfWorkers; i++ { + go s.worker(q, i, done, s.killSignal) + } + + // wait until scan of all images is done - non blocking. once all done, notify on fullScanDone chan + go func() { + for c := 0; c < len(imageIDToScanData); c++ { + select { + case <-done: + atomic.AddUint32(imagesCompletedToScan, 1) + case <-s.killSignal: + log.WithFields(s.logFields).Debugf("Scan process was canceled - stop waiting for finished jobs") + return + } + } + + fullScanDone <- true + }() + + // send all scan data on scan data queue, for workers to pick it up. + for _, data := range imageIDToScanData { + go func(data *scanData, ks chan bool) { + select { + case q <- data: + atomic.AddUint32(imagesStartedToScan, 1) + case <-ks: + log.WithFields(s.logFields).Debugf("Scan process was canceled. imageID=%v, scanUUID=%v", data.instance.ID, data.scanUUID) + return + } + }(data, s.killSignal) + } + + // wait for killSignal or fullScanDone + select { + case <-s.killSignal: + log.WithFields(s.logFields).Info("Scan process was canceled") + case <-fullScanDone: + log.WithFields(s.logFields).Infof("All jobs has finished") + // Nonblocking notification of a finished scan + nonBlockingNotification(scanDone) + } +} + +// worker waits for data on the queue, runs a scan job and waits for results from that scan job. Upon completion, done is notified to the caller. +func (s *Scanner) worker(queue chan *scanData, workNumber int, done, ks chan bool) { + for { + select { + case data := <-queue: + job, err := s.runJob(data) + if err != nil { + errMsg := fmt.Errorf("failed to run job: %v", err) + log.WithFields(s.logFields).Error(errMsg) + s.Lock() + data.success = false + data.scanErr = &types2.ScanError{ + ErrMsg: err.Error(), + ErrType: string(types2.JobRun), + ErrSource: types2.ScanErrSourceJob, + } + data.completed = true + s.Unlock() + } else { + s.waitForResult(data, ks) + } + + s.deleteJobIfNeeded(&job, data.success, data.completed) + + select { + case done <- true: + case <-ks: + log.WithFields(s.logFields).Infof("Image scan was canceled. imageID=%v", data.instance.ID) + } + case <-ks: + log.WithFields(s.logFields).Debugf("worker #%v halted", workNumber) + return + } + } +} + +func (s *Scanner) waitForResult(data *scanData, ks chan bool) { + //log.WithFields(s.logFields).Infof("Waiting for result. imageID=%+v", data.imageID) + ticker := time.NewTicker(s.scanConfig.JobResultTimeout) + select { + case <-data.resultChan: + log.WithFields(s.logFields).Infof("Instance scanned result has arrived. instanceID=%v", data.instance.ID) + case <-ticker.C: + errMsg := fmt.Errorf("job has timed out. imageID=%v", data.instance.ID) + log.WithFields(s.logFields).Warn(errMsg) + s.Lock() + data.success = false + data.scanErr = &types2.ScanError{ + ErrMsg: errMsg.Error(), + ErrType: string(types2.JobTimeout), + ErrSource: types2.ScanErrSourceJob, + } + data.timeout = true + data.completed = true + s.Unlock() + case <-ks: + log.WithFields(s.logFields).Infof("Image scan was canceled. imageID=%v", data.instance.ID) + } +} + +func (s *Scanner) runJob(data *scanData) (types2.Job, error) { + rootVolume, err := s.providerClient.GetInstanceRootVolume(data.instance) + if err != nil { + return types2.Job{}, fmt.Errorf("failed to get instance root volume. instance id=%v: %v", data.instance.ID, err) + } + + // create a snapshot of that vm + srcSnapshot, err := s.providerClient.CreateSnapshot(rootVolume) + if err != nil { + return types2.Job{}, fmt.Errorf("failed to create snapshot: %v", err) + } + if err := s.providerClient.WaitForSnapshotReady(srcSnapshot); err != nil { + return types2.Job{}, err + } + + //copy the snapshot to the scanner region + // TODO make sure we need this. + cpySnapshot, err := s.providerClient.CopySnapshot(srcSnapshot, s.region) + if err != nil { + return types2.Job{}, fmt.Errorf("failed to copy snapshot: %v", err) + } + if err := s.providerClient.WaitForSnapshotReady(cpySnapshot); err != nil { + return types2.Job{}, err + } + + // create the scanner job (vm) with a boot script + launchedInstance, err := s.providerClient.LaunchInstance(s.amiID, "", cpySnapshot) + if err != nil { + return types2.Job{}, fmt.Errorf("failed to launch instance: %v", err) + } + + return types2.Job{ + Instance: launchedInstance, + SrcSnapshot: srcSnapshot, + DstSnapshot: cpySnapshot, + }, nil +} + +func (s *Scanner) deleteJobIfNeeded(job *types2.Job, isSuccessfulJob, isCompletedJob bool) { + if job == nil { + return + } + + // delete uncompleted jobs - scan process was canceled + if !isCompletedJob { + s.deleteJob(job) + return + } + + switch s.scanConfig.DeleteJobPolicy { + case config.DeleteJobPolicyNever: + // do nothing + case config.DeleteJobPolicyAll: + s.deleteJob(job) + case config.DeleteJobPolicySuccessful: + if isSuccessfulJob { + s.deleteJob(job) + } + } +} + +func (s *Scanner) deleteJob(job *types2.Job) { + if err := s.providerClient.DeleteInstance(job.Instance); err != nil { + log.Errorf("failed to delete instance: %v", err) + } + if err := s.providerClient.DeleteSnapshot(job.SrcSnapshot); err != nil { + log.Errorf("failed to delete source snapshot: %v", err) + } + if err := s.providerClient.DeleteSnapshot(job.DstSnapshot); err != nil { + log.Errorf("failed to delete dest snapshot: %v", err) + } +} + +// Due to K8s names constraint we will take the image name w/o the tag and repo. +//func getSimpleImageName(imageName string) (string, error) { +// ref, err := reference.ParseNormalizedNamed(imageName) +// if err != nil { +// return "", fmt.Errorf("failed to parse image name. name=%v: %v", imageName, err) +// } +// +// refName := ref.Name() +// // Take only image name from repo path (ex. solsson/kafka ==> kafka) +// repoEnd := strings.LastIndex(refName, "/") +// +// return refName[repoEnd+1:], nil +//} + +// Job names require their names to follow the DNS label standard as defined in RFC 1123 +// Note: job name is added as a label to the pod template spec so it should follow the DNS label standard and not just DNS-1123 subdomain +// +// This means the name must: +// * contain at most 63 characters +// * contain only lowercase alphanumeric characters or ‘-’ +// * start with an alphanumeric character +// * end with an alphanumeric character. +//func createJobName(imageName string) (string, error) { +// //simpleName, err := getSimpleImageName(imageName) +// //if err != nil { +// // return "", err +// //} +// +// jobName := "scanner-" + simpleName + "-" + uuid.NewV4().String() +// +// // contain at most 63 characters +// jobName = stringsutils.TruncateString(jobName, k8s.MaxK8sJobName) +// +// // contain only lowercase alphanumeric characters or ‘-’ +// jobName = strings.ToLower(jobName) +// jobName = strings.ReplaceAll(jobName, "_", "-") +// +// // no need to validate start, we are using 'jobName' +// +// // end with an alphanumeric character +// jobName = strings.TrimRight(jobName, "-") +// +// return jobName, nil +//} + +//func (s *Scanner) createJob(data *scanData) (*batchv1.Job, error) { +// // We will scan each image once, based on the first pod context. The result will be applied for all other pods with this image. +// podContext := data.contexts[0] +// +// jobName, err := createJobName(podContext.imageName) +// if err != nil { +// return nil, fmt.Errorf("failed to create job name. namespace=%v, pod=%v, container=%v, image=%v, hash=%v: %v", +// podContext.namespace, podContext.podName, podContext.containerName, podContext.imageName, data.imageHash, err) +// } +// +// // Set job values on scanner job template +// job := s.scannerJobTemplate.DeepCopy() +// if !data.shouldScanCISDockerBenchmark { +// removeCISDockerBenchmarkScannerFromJob(job) +// } +// job.SetName(jobName) +// job.SetNamespace(podContext.namespace) +// setJobScanUUID(job, data.scanUUID) +// setJobImageIDToScan(job, data.imageID) +// setJobImageHashToScan(job, data.imageHash) +// setJobImageNameToScan(job, podContext.imageName) +// if podContext.imagePullSecret != "" { +// log.WithFields(s.logFields).Debugf("Adding private registry credentials to image: %s", podContext.imageName) +// setJobDockerConfigFromImagePullSecret(job, podContext.imagePullSecret) +// } else { +// // Use private repo sa credentials only if there is no imagePullSecret +// for _, adder := range s.credentialAdders { +// if adder.ShouldAdd() { +// adder.Add(job) +// } +// } +// } +// +// return job, nil +//} +// +//func removeCISDockerBenchmarkScannerFromJob(job *batchv1.Job) { +// var containers []corev1.Container +// for i := range job.Spec.Template.Spec.Containers { +// container := job.Spec.Template.Spec.Containers[i] +// if container.Name != cisDockerBenchmarkScannerContainerName { +// containers = append(containers, container) +// } +// } +// job.Spec.Template.Spec.Containers = containers +//} + +// Create docker config from imagePullSecret that contains the username and the password required to pull the image. +// We need to do the following: +// 1. Create a volume that holds the `secretName` data +// 2. Mount the volume into each container to a specific path (`BasicVolumeMountPath`/`DockerConfigFileName`) +// 3. Set `DOCKER_CONFIG` to point to the directory that contains the config.json. +//func setJobDockerConfigFromImagePullSecret(job *batchv1.Job, secretName string) { +// job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes, corev1.Volume{ +// Name: _creds.BasicVolumeName, +// VolumeSource: corev1.VolumeSource{ +// Secret: &corev1.SecretVolumeSource{ +// SecretName: secretName, +// Items: []corev1.KeyToPath{ +// { +// Key: corev1.DockerConfigJsonKey, +// Path: _creds.DockerConfigFileName, +// }, +// }, +// }, +// }, +// }) +// for i := range job.Spec.Template.Spec.Containers { +// container := &job.Spec.Template.Spec.Containers[i] +// container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ +// Name: _creds.BasicVolumeName, +// ReadOnly: true, +// MountPath: _creds.BasicVolumeMountPath, +// }) +// container.Env = append(container.Env, corev1.EnvVar{ +// Name: _creds.DockerConfigEnvVar, +// Value: _creds.BasicVolumeMountPath, +// }) +// } +//} + +//func setJobImageIDToScan(job *batchv1.Job, imageID string) { +// for i := range job.Spec.Template.Spec.Containers { +// container := &job.Spec.Template.Spec.Containers[i] +// container.Env = append(container.Env, corev1.EnvVar{Name: shared.ImageIDToScan, Value: imageID}) +// } +//} +// +//func setJobImageHashToScan(job *batchv1.Job, imageHash string) { +// for i := range job.Spec.Template.Spec.Containers { +// container := &job.Spec.Template.Spec.Containers[i] +// container.Env = append(container.Env, corev1.EnvVar{Name: shared.ImageHashToScan, Value: imageHash}) +// } +//} +// +//func setJobImageNameToScan(job *batchv1.Job, imageName string) { +// for i := range job.Spec.Template.Spec.Containers { +// container := &job.Spec.Template.Spec.Containers[i] +// container.Env = append(container.Env, corev1.EnvVar{Name: shared.ImageNameToScan, Value: imageName}) +// } +//} +// +//func setJobScanUUID(job *batchv1.Job, scanUUID string) { +// for i := range job.Spec.Template.Spec.Containers { +// container := &job.Spec.Template.Spec.Containers[i] +// container.Env = append(container.Env, corev1.EnvVar{Name: shared.ScanUUID, Value: scanUUID}) +// } +//} diff --git a/runtime_scan/pkg/scanner/scanner.go b/runtime_scan/pkg/scanner/scanner.go new file mode 100644 index 000000000..6207ccb27 --- /dev/null +++ b/runtime_scan/pkg/scanner/scanner.go @@ -0,0 +1,206 @@ +// Copyright © 2022 Cisco Systems, Inc. and its affiliates. +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scanner + +import ( + "fmt" + "go/scanner" + "sync" + + uuid "github.com/satori/go.uuid" + log "github.com/sirupsen/logrus" + + _config "github.com/openclarity/vmclarity/runtime_scan/pkg/config" + "github.com/openclarity/vmclarity/runtime_scan/pkg/provider" + "github.com/openclarity/vmclarity/runtime_scan/pkg/types" +) + +type Scanner struct { + instanceIDToScanData map[string]*scanData + providerClient provider.Client + scanConfig *_config.ScanConfig + killSignal chan bool + progress types.ScanProgress + logFields log.Fields + + region string + vpcID string + subnetID string + amiID string + + sync.Mutex +} + +type scanData struct { + instance types.Instance + //contexts []*imagePodContext // All the pods that contain this image hash + scanUUID string + vulnerabilitiesResult vulnerabilitiesScanResult + //cisDockerBenchmarkResult cisDockerBenchmarkScanResult + //shouldScanCISDockerBenchmark bool + resultChan chan bool + success bool + completed bool + timeout bool + scanErr *types.ScanError +} + +type vulnerabilitiesScanResult struct { + result []string + //layerCommands []*models.ResourceLayerCommand + success bool + completed bool + error *scanner.Error +} + +func CreateScanner(config *_config.Config, providerClient provider.Client) *Scanner { + s := &Scanner{ + instanceIDToScanData: nil, + providerClient: providerClient, + scanConfig: nil, + killSignal: nil, + progress: types.ScanProgress{}, + logFields: nil, + region: config.Region, + vpcID: config.VpcID, + subnetID: config.SubnetID, + amiID: config.AmiID, + Mutex: sync.Mutex{}, + } + + return s +} + +// initScan Calculate properties of scan targets +// nolint:cyclop +func (s *Scanner) initScan() error { + //// Get all target pods + //for _, namespace := range s.scanConfig.Instances { + // podList, err := s.clientset.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{}) + // if err != nil { + // return fmt.Errorf("failed to list pods. namespace=%s: %v", namespace, err) + // } + // podsToScan = append(podsToScan, podList.Items...) + // if namespace == corev1.NamespaceAll { + // break + // } + //} + + imageIDToScanData := make(map[string]*scanData) + + // Populate the image to scanData map from all target pods + for _, instance := range s.scanConfig.Instances { + //if s.shouldIgnorePod(&podsToScan[i]) { + // continue + //} + + // TODO: (idanf) verify if we need to read pod image pull secrets or just mount it to the scanner job + //secrets := k8sutils.GetPodImagePullSecrets(s.clientset, pod) + + // Due to scenarios where image name in the `pod.Status.ContainerStatuses` is different + // from image name in the `pod.Spec.Containers` we will take only image id from `pod.Status.ContainerStatuses`. + //containerNameToImageID := make(map[string]string) + //for _, container := range append(pod.Status.ContainerStatuses, pod.Status.InitContainerStatuses...) { + // containerNameToImageID[container.Name] = k8sutils.NormalizeImageID(container.ImageID) + //} + + //containers := append(pod.Spec.Containers, pod.Spec.InitContainers...) + + //for _, container := range containers { + //imageID, ok := containerNameToImageID[container.Name] + //if !ok { + // log.Warnf("Image id is missing. pod=%v, namepspace=%v, container=%v ,image=%v", + // pod.GetName(), pod.GetNamespace(), container.Name, container.Image) + // continue + //} + //imageHash := k8sutils.ParseImageHash(imageID) + //if imageHash == "" { + // log.WithFields(s.logFields).Warnf("Failed to get image hash - ignoring image. "+ + // "pod=%v, namepspace=%v, image name=%v", pod.GetName(), pod.GetNamespace(), container.Image) + // continue + //} + //// Create pod context + //podContext := &imagePodContext{ + // containerName: container.Name, + // podName: pod.GetName(), + // namespace: pod.GetNamespace(), + // imagePullSecret: k8sutils.GetMatchingSecretName(secrets, container.Image), + // imageName: container.Image, + // podUID: string(pod.GetUID()), + // podLabels: labels.Set(pod.GetLabels()), + //} + //if data, ok := instanceIDToScanData[imageID]; !ok { + // Image added for the first time, create scan data and append pod context + imageIDToScanData[instance.ID] = &scanData{ + instance: instance, + scanUUID: uuid.NewV4().String(), + vulnerabilitiesResult: vulnerabilitiesScanResult{}, + //shouldScanCISDockerBenchmark: s.scanConfig.ShouldScanCISDockerBenchmark, + resultChan: make(chan bool), + success: false, + completed: false, + timeout: false, + scanErr: nil, + } + //} else { + // Image already exist in map, just append the pod context + //data.contexts = append(data.contexts, podContext) + //} + } + //} + + s.instanceIDToScanData = imageIDToScanData + s.progress.ImagesToScan = uint32(len(imageIDToScanData)) + + log.WithFields(s.logFields).Infof("Total %d unique images to scan", s.progress.ImagesToScan) + + return nil +} + +func (s *Scanner) Scan(scanConfig *_config.ScanConfig, scanDone chan struct{}) error { + s.Lock() + defer s.Unlock() + + s.scanConfig = scanConfig + + log.WithFields(s.logFields).Infof("Start scanning...") + + s.progress.Status = types.ScanInit + + if err := s.initScan(); err != nil { + s.progress.SetStatus(types.ScanInitFailure) + return fmt.Errorf("failed to initiate scan: %v", err) + } + + if s.progress.ImagesToScan == 0 { + log.WithFields(s.logFields).Info("Nothing to scan") + s.progress.SetStatus(types.NothingToScan) + nonBlockingNotification(scanDone) + return nil + } + + s.progress.SetStatus(types.Scanning) + go func() { + s.jobBatchManagement(scanDone) + + s.Lock() + s.progress.SetStatus(types.DoneScanning) + s.Unlock() + }() + + + return nil +} diff --git a/runtime_scan/pkg/types/errors.go b/runtime_scan/pkg/types/errors.go new file mode 100644 index 000000000..40396cc53 --- /dev/null +++ b/runtime_scan/pkg/types/errors.go @@ -0,0 +1,35 @@ +// Copyright © 2022 Cisco Systems, Inc. and its affiliates. +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +type ScanError struct { + ErrMsg string + ErrType string + ErrSource ScanErrorSource +} + +type ScanErrorType string + +const ( + JobRun ScanErrorType = "errorJobRun" + JobTimeout ScanErrorType = "errorJobTimeout" +) + +type ScanErrorSource string + +const ( + ScanErrSourceJob ScanErrorSource = "ScanErrSourceJob" +) diff --git a/runtime_scan/pkg/types/types.go b/runtime_scan/pkg/types/types.go new file mode 100644 index 000000000..fded8a56f --- /dev/null +++ b/runtime_scan/pkg/types/types.go @@ -0,0 +1,85 @@ +package types + +type Tag struct { + Key string + Val string +} + +type SecurityGroup struct { + ID string +} + +type VPC struct { + ID string + SecurityGroups []SecurityGroup +} + +type Region struct { + ID string + VPCs []VPC +} + +type ScanScope struct { + All bool + Regions []Region + ScanStopped bool + IncludeTags []*Tag + ExcludeTags []*Tag +} + +type Status string + +const ( + Idle Status = "Idle" + ScanInit Status = "ScanInit" + ScanInitFailure Status = "ScanInitFailure" + NothingToScan Status = "NothingToScan" + Scanning Status = "Scanning" + DoneScanning Status = "DoneScanning" +) + +type ScanProgress struct { + ImagesToScan uint32 + ImagesStartedToScan uint32 + ImagesCompletedToScan uint32 + Status Status +} + +func (s *ScanProgress) SetStatus(status Status) { + s.Status = status +} + +type VMScanResult struct { + // VM data + VMID string + // Scan results + Vulnerabilities []string + Success bool +} + +type ScanResults struct { + ImageScanResults []*VMScanResult + Progress ScanProgress +} + +type Job struct { + Instance Instance + SrcSnapshot Snapshot + DstSnapshot Snapshot +} + +type Instance struct { + ID string + Region string +} + +type Snapshot struct { + ID string + Region string +} + +type Volume struct { + ID string + Name string + Region string +} From 71358626cb7102b74d0a63103e29342f480186bd Mon Sep 17 00:00:00 2001 From: Erez Fishhimer Date: Fri, 11 Nov 2022 01:45:07 +0200 Subject: [PATCH 02/23] ready --- runtime_scan/go.mod | 36 +- runtime_scan/go.sum | 95 ---- runtime_scan/pkg/config/config.go | 16 +- runtime_scan/pkg/orchestrator/orchestrator.go | 53 +- runtime_scan/pkg/provider/aws/client.go | 247 ++++++--- runtime_scan/pkg/provider/aws/client_test.go | 469 ++++++++++++++++++ runtime_scan/pkg/provider/provider.go | 27 +- runtime_scan/pkg/scanner/job_managment.go | 215 ++------ runtime_scan/pkg/scanner/scanner.go | 159 +++--- runtime_scan/pkg/types/types.go | 59 +-- runtime_scan/pkg/utils/utils.go | 16 + 11 files changed, 832 insertions(+), 560 deletions(-) create mode 100644 runtime_scan/pkg/utils/utils.go diff --git a/runtime_scan/go.mod b/runtime_scan/go.mod index 6da0cd59f..3e3fae07c 100644 --- a/runtime_scan/go.mod +++ b/runtime_scan/go.mod @@ -8,12 +8,9 @@ require ( github.com/satori/go.uuid v1.2.0 github.com/sirupsen/logrus v1.9.0 github.com/spf13/viper v1.14.0 - k8s.io/client-go v0.25.3 ) require ( - github.com/PuerkitoBio/purell v1.1.1 // indirect - github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/aws/aws-sdk-go-v2 v1.17.1 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.12.23 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19 // indirect @@ -25,27 +22,12 @@ require ( github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.17.1 // indirect github.com/aws/smithy-go v1.13.4 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/emicklei/go-restful/v3 v3.8.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect - github.com/go-logr/logr v1.2.3 // indirect - github.com/go-openapi/jsonpointer v0.19.5 // indirect - github.com/go-openapi/jsonreference v0.19.5 // indirect - github.com/go-openapi/swag v0.19.14 // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.2 // indirect - github.com/google/gnostic v0.5.7-v3refs // indirect - github.com/google/gofuzz v1.1.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/josharian/intern v1.0.0 // indirect - github.com/json-iterator/go v1.1.12 // indirect github.com/magiconair/properties v1.8.6 // indirect - github.com/mailru/easyjson v0.7.6 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.0.5 // indirect github.com/spf13/afero v1.9.2 // indirect @@ -53,24 +35,10 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.4.1 // indirect - golang.org/x/net v0.0.0-20221014081412-f15817d10f9b // indirect - golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect golang.org/x/sys v0.0.0-20220908164124-27713097b956 // indirect - golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect golang.org/x/text v0.4.0 // indirect - golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.28.1 // indirect - gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/api v0.25.3 // indirect - k8s.io/apimachinery v0.25.3 // indirect - k8s.io/klog/v2 v2.70.1 // indirect - k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect - k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed // indirect - sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect - sigs.k8s.io/yaml v1.2.0 // indirect ) diff --git a/runtime_scan/go.sum b/runtime_scan/go.sum index eeb250779..897863d7a 100644 --- a/runtime_scan/go.sum +++ b/runtime_scan/go.sum @@ -38,10 +38,6 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= -github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/aws/aws-sdk-go-v2 v1.17.1 h1:02c72fDJr87N8RAC2s3Qu0YuvMRZKNZJ9F+lAehCazk= github.com/aws/aws-sdk-go-v2 v1.17.1/go.mod h1:JLnGeGONAyi2lWXI1p0PCIOIy333JMVK1U7Hf0aRFLw= github.com/aws/aws-sdk-go-v2/config v1.17.10 h1:zBy5QQ/mkvHElM1rygHPAzuH+sl8nsdSaxSWj0+rpdE= @@ -76,13 +72,9 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/emicklei/go-restful/v3 v3.8.0 h1:eCZ8ulSerjdAiaNpF7GxXIE7ZCMo1moN1qX+S609eVw= -github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -95,20 +87,6 @@ github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbS github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.19.5 h1:1WJP/wi4OjB4iV8KVbH73rQaoialJrqv8gitZLxGLtM= -github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= -github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -134,13 +112,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= -github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -150,12 +123,8 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= -github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -184,41 +153,21 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= -github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/onsi/ginkgo/v2 v2.1.6 h1:Fx2POJZfKRQcM1pH49qSZiYeu319wji004qX+GDovrU= -github.com/onsi/gomega v1.20.1 h1:PA/3qinGoukvymdIDV8pii6tiZgC8kbmJO6Z5+b002Q= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.0.5 h1:ipoSadvV8oGUjnUbMub59IDPPwfxF694nG/jwbMiyQg= @@ -234,7 +183,6 @@ github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.9.2 h1:j49Hj62F0n+DaZ1dDCvhABaPNSGNkt32oRFxI33IEMw= github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= @@ -245,14 +193,11 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.14.0 h1:Rg7d3Lo706X9tHsJMUjdiwMpHB7W8WnSVOssIY+JElU= github.com/spf13/viper v1.14.0/go.mod h1:WT//axPky3FdvXHzGw33dNdXXXfFQqmEalje+egj8As= -github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -321,7 +266,6 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -341,8 +285,6 @@ golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20221014081412-f15817d10f9b h1:tvrvnPFcdzp294diPnrdZZZ8XUt2Tyj7svb7X52iDuU= -golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -352,8 +294,6 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 h1:nt+Q6cXKz4MosCSpnbMtqiQ8Oz0pxTef2B4Vca2lvfk= -golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -402,8 +342,6 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220908164124-27713097b956 h1:XeJjHH1KiLpKGb6lvMiksZ9l0fVUh+AmGcm0nOMEBOY= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -415,8 +353,6 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220609170525-579cf78fd858 h1:Dpdu/EMxGMFgq0CeYMh4fazTD2vtlZRYE7wyynxJb9U= -golang.org/x/time v0.0.0-20220609170525-579cf78fd858/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -454,7 +390,6 @@ golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= @@ -463,7 +398,6 @@ golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -495,7 +429,6 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -527,7 +460,6 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -560,18 +492,11 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= -gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -579,7 +504,6 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -589,25 +513,6 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.25.3 h1:Q1v5UFfYe87vi5H7NU0p4RXC26PPMT8KOpr1TLQbCMQ= -k8s.io/api v0.25.3/go.mod h1:o42gKscFrEVjHdQnyRenACrMtbuJsVdP+WVjqejfzmI= -k8s.io/apimachinery v0.25.3 h1:7o9ium4uyUOM76t6aunP0nZuex7gDf8VGwkR5RcJnQc= -k8s.io/apimachinery v0.25.3/go.mod h1:jaF9C/iPNM1FuLl7Zuy5b9v+n35HGSh6AQ4HYRkCqwo= -k8s.io/client-go v0.25.3 h1:oB4Dyl8d6UbfDHD8Bv8evKylzs3BXzzufLiO27xuPs0= -k8s.io/client-go v0.25.3/go.mod h1:t39LPczAIMwycjcXkVc+CB+PZV69jQuNx4um5ORDjQA= -k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= -k8s.io/klog/v2 v2.70.1 h1:7aaoSdahviPmR+XkS7FyxlkkXs6tHISSG03RxleQAVQ= -k8s.io/klog/v2 v2.70.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 h1:MQ8BAZPZlWk3S9K4a9NCkIFQtZShWqoha7snGixVgEA= -k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1/go.mod h1:C/N6wCaBHeBHkHUesQOQy2/MZqGgMAFPqGsGQLdbZBU= -k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed h1:jAne/RjBTyawwAy0utX5eqigAwz/lQhTmy+Hr/Cpue4= -k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= -sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= -sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/runtime_scan/pkg/config/config.go b/runtime_scan/pkg/config/config.go index 5e5210513..ae01af570 100644 --- a/runtime_scan/pkg/config/config.go +++ b/runtime_scan/pkg/config/config.go @@ -17,43 +17,31 @@ package config import ( "github.com/spf13/viper" - "k8s.io/client-go/kubernetes" ) const ( ScannerJobResultListenPort = "SCANNER_JOB_RESULT_LISTEN_PORT" - CredsSecretNamespace = "CREDS_SECRET_NAMESPACE" // nolint: gosec - ScannerJobTemplateConfigMapName = "SCANNER_JOB_TEMPLATE_CONFIG_MAP_NAME" - ScannerJobTemplateConfigMapNamespace = "SCANNER_JOB_TEMPLATE_CONFIG_MAP_NAMESPACE" defaultScannerJobResultListenPort = 8888 ) type Config struct { ScannerJobResultListenPort int - CredsSecretNamespace string Region string - VpcID string - SubnetID string AmiID string } func setConfigDefaults() { - viper.SetDefault(CredsSecretNamespace, "kubeclarity") - viper.SetDefault(ScannerJobTemplateConfigMapName, "") - viper.SetDefault(ScannerJobTemplateConfigMapNamespace, "kubeclarity") + // TODO defaults for region and ami ID viper.SetDefault(ScannerJobResultListenPort, defaultScannerJobResultListenPort) viper.AutomaticEnv() } -func LoadConfig(clientset kubernetes.Interface) (*Config, error) { +func LoadConfig() (*Config, error) { setConfigDefaults() - - config := &Config{ ScannerJobResultListenPort: viper.GetInt(ScannerJobResultListenPort), - CredsSecretNamespace: viper.GetString(CredsSecretNamespace), } return config, nil diff --git a/runtime_scan/pkg/orchestrator/orchestrator.go b/runtime_scan/pkg/orchestrator/orchestrator.go index d62d2646e..0ccb6b0ce 100644 --- a/runtime_scan/pkg/orchestrator/orchestrator.go +++ b/runtime_scan/pkg/orchestrator/orchestrator.go @@ -23,35 +23,38 @@ import ( _config "github.com/openclarity/vmclarity/runtime_scan/pkg/config" "github.com/openclarity/vmclarity/runtime_scan/pkg/provider" - aws2 "github.com/openclarity/vmclarity/runtime_scan/pkg/provider/aws" + "github.com/openclarity/vmclarity/runtime_scan/pkg/provider/aws" _scanner "github.com/openclarity/vmclarity/runtime_scan/pkg/scanner" + "github.com/openclarity/vmclarity/runtime_scan/pkg/types" ) type Orchestrator struct { scanner *_scanner.Scanner config *_config.Config providerClient provider.Client + // server *rest.Server sync.Mutex } -//go:generate $GOPATH/bin/mockgen -destination=./mock_orchestrator.go -package=orchestrator github.com/openclarity/kubeclarity/runtime_scan/pkg/orchestrator VulnerabilitiesScanner +//go:generate $GOPATH/bin/mockgen -destination=./mock_orchestrator.go -package=orchestrator github.com/openclarity/vmclarity/runtime_scan/pkg/orchestrator VulnerabilitiesScanner type VulnerabilitiesScanner interface { Start(errChan chan struct{}) Scan(scanConfig *_config.ScanConfig, scanDone chan struct{}) error - //ScanProgress() types.ScanProgress - ///Results() *types.ScanResults - //Clear() + ScanProgress() types.ScanProgress + Results() *types.ScanResults + Clear() Stop() } func Create(config *_config.Config) (*Orchestrator, error) { // for now will statically create aws client here (until we support more cloud providers) - awsClient, err := aws2.Create() + awsClient, err := aws.Create() if err != nil { return nil, fmt.Errorf("failed to create aws client: %v", err) } orc := &Orchestrator{ + scanner: _scanner.CreateScanner(config, awsClient), config: config, providerClient: awsClient, Mutex: sync.Mutex{}, @@ -63,10 +66,14 @@ func Create(config *_config.Config) (*Orchestrator, error) { func (o *Orchestrator) Start(errChan chan struct{}) { // Start result server log.Infof("Starting Orchestrator server") + // o.server.Start(errChan) } func (o *Orchestrator) Stop() { + o.Clear() + log.Infof("Stopping Orchestrator server") + //o.server.Stop() } func (o *Orchestrator) Scan(scanConfig *_config.ScanConfig, scanDone chan struct{}) error { @@ -83,23 +90,25 @@ func (o *Orchestrator) Scan(scanConfig *_config.ScanConfig, scanDone chan struct return nil } -// -//func (o *Orchestrator) ScanProgress() types.ScanProgress { -// return o.getScanner().ScanProgress() -//} -// -//func (o *Orchestrator) Results() *types.ScanResults { -// return o.getScanner().Results() -//} -//func (o *Orchestrator) Clear() { -// o.Lock() -// defer o.Unlock() -// -// log.Infof("Clearing Orchestrator") -// o.scanner.Clear() -// //o.scanner = _scanner.CreateScanner(o.config, o.clientset) -//} +func (o *Orchestrator) ScanProgress() types.ScanProgress { + return o.getScanner().ScanProgress() +} + +func (o *Orchestrator) Results() *types.ScanResults { + return nil + // TODO + //return o.getScanner().Results() +} + +func (o *Orchestrator) Clear() { + o.Lock() + defer o.Unlock() + + log.Infof("Clearing Orchestrator") + o.scanner.Clear() + o.scanner = _scanner.CreateScanner(o.config, o.providerClient) +} func (o *Orchestrator) getScanner() *_scanner.Scanner { o.Lock() diff --git a/runtime_scan/pkg/provider/aws/client.go b/runtime_scan/pkg/provider/aws/client.go index f0b04aad0..6f822a2bd 100644 --- a/runtime_scan/pkg/provider/aws/client.go +++ b/runtime_scan/pkg/provider/aws/client.go @@ -9,10 +9,16 @@ import ( awsconfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/ec2" ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + log "github.com/sirupsen/logrus" "github.com/openclarity/vmclarity/runtime_scan/pkg/types" + "github.com/openclarity/vmclarity/runtime_scan/pkg/utils" ) +type Client struct { + ec2Client *ec2.Client +} + var ( snapshotDescription = "VMClarity snapshot" tagKey = "Owner" @@ -25,10 +31,6 @@ var ( } ) -type Client struct { - ec2Client *ec2.Client -} - func Create() (*Client, error) { var awsClient Client @@ -44,6 +46,7 @@ func Create() (*Client, error) { func (c *Client) Discover(scope *types.ScanScope) ([]types.Instance, error) { var ret []types.Instance + var filters []ec2types.Filter regions, err := c.getRegionsToScan(scope) if err != nil { @@ -52,32 +55,69 @@ func (c *Client) Discover(scope *types.ScanScope) ([]types.Instance, error) { if len(regions) == 0 { return nil, fmt.Errorf("no regions to scan") } + filters = append(filters, createInclusionTagsFilters(scope.IncludeTags)...) + filters = append(filters, createInstanceStateFilters(scope.ScanStopped)...) - for _, region := range regions { - describeFilters := createDescribeFilters(scope, region) + for _, regionID := range regions { + region, err := findRegionByID(scope.Regions, regionID) + if err != nil { + log.Errorf("failed to find region by ID: %v", err) + continue + } + + // if no vpcs, that mean that we don't need any vpc filters + if len(region.VPCs) == 0 { + instances, err := c.GetInstances(filters, scope.ExcludeTags, regionID) + if err != nil { + return nil, fmt.Errorf("failed to get instances: %v", err) + } + ret = append(ret, instances...) + continue + } + + // need to do a per vpc call for DescribeInstances + for _, vpc := range region.VPCs { + filters = append(filters, createVPCFilters(vpc)...) + + instances, err := c.GetInstances(filters, scope.ExcludeTags, regionID) + if err != nil { + return nil, fmt.Errorf("failed to get instances: %v", err) + } + ret = append(ret, instances...) + } + } + return ret, nil +} + +func (c *Client) GetInstances(filters []ec2types.Filter, excludeTags []*types.Tag, regionID string) ([]types.Instance, error) { + var ret []types.Instance - out, err := c.ec2Client.DescribeInstances(context.TODO(), &ec2.DescribeInstancesInput{ - Filters: describeFilters, - MaxResults: nil, // TODO - NextToken: nil, // TODO + out, err := c.ec2Client.DescribeInstances(context.TODO(), &ec2.DescribeInstancesInput{ + Filters: filters, + MaxResults: utils.Int32Ptr(50), // TODO what will be a good number? + }, func(options *ec2.Options) { + options.Region = regionID + }) + if err != nil { + return nil, fmt.Errorf("failed to describe instances: %v", err) + } + ret = append(ret, getInstancesFromDescribeInstancesOutput(out, excludeTags, regionID)...) + + // use pagination + for out.NextToken != nil { + out, err = c.ec2Client.DescribeInstances(context.TODO(), &ec2.DescribeInstancesInput{ + Filters: filters, + MaxResults: utils.Int32Ptr(50), + NextToken: out.NextToken, }, func(options *ec2.Options) { - options.Region = region + options.Region = regionID }) if err != nil { return nil, fmt.Errorf("failed to describe instances: %v", err) } - // get all returned instances: - for _, reservation := range out.Reservations { - for _, instance := range reservation.Instances { - if instance.InstanceId != nil { - ret = append(ret, types.Instance{ - ID: *instance.InstanceId, - Region: region, - }) - } - } - } + ret = append(ret, getInstancesFromDescribeInstancesOutput(out, excludeTags, regionID)...) } + return ret, nil } @@ -105,9 +145,9 @@ func (c *Client) CreateSnapshot(volume types.Volume) (types.Snapshot, error) { } func (c *Client) WaitForSnapshotReady(snapshot types.Snapshot) error { - ticker := time.NewTicker(2 * time.Second) + ticker := time.NewTicker(3 * time.Second) defer ticker.Stop() - timeout := time.After(2 * time.Minute) + timeout := time.After(3 * time.Minute) for { select { @@ -118,7 +158,10 @@ func (c *Client) WaitForSnapshotReady(snapshot types.Snapshot) error { options.Region = snapshot.Region }) if err != nil { - return fmt.Errorf("failed to descrive snapshot: %v", err) + return fmt.Errorf("failed to describe snapshot: %v", err) + } + if len(out.Snapshots) != 1 { + return fmt.Errorf("got unexcpected number of snapshots (%v) with snapshot id %v. excpecting 1", len(out.Snapshots), snapshot.ID) } if out.Snapshots[0].State == ec2types.SnapshotStateCompleted { return nil @@ -131,9 +174,9 @@ func (c *Client) WaitForSnapshotReady(snapshot types.Snapshot) error { func (c *Client) CopySnapshot(snapshot types.Snapshot, dstRegion string) (types.Snapshot, error) { snap, err := c.ec2Client.CopySnapshot(context.TODO(), &ec2.CopySnapshotInput{ - SourceRegion: &snapshot.Region, - SourceSnapshotId: &snapshot.ID, - Description: &snapshotDescription, + SourceRegion: &snapshot.Region, + SourceSnapshotId: &snapshot.ID, + Description: &snapshotDescription, TagSpecifications: []ec2types.TagSpecification{ { ResourceType: ec2types.ResourceTypeSnapshot, @@ -179,6 +222,7 @@ func (c *Client) GetInstanceRootVolume(instance types.Instance) (types.Volume, e outInstance := out.Reservations[0].Instances[0] rootDeviceName := *outInstance.RootDeviceName + // find root volume of the instance for _, blkDevice := range outInstance.BlockDeviceMappings { if strings.Compare(*blkDevice.DeviceName, rootDeviceName) == 0 { return types.Volume{ @@ -192,31 +236,33 @@ func (c *Client) GetInstanceRootVolume(instance types.Instance) (types.Volume, e } func (c *Client) LaunchInstance(ami, deviceName string, snapshot types.Snapshot) (types.Instance, error) { - var max = int32(1) - var min = int32(1) - out, err := c.ec2Client.RunInstances(context.TODO(), &ec2.RunInstancesInput{ - MaxCount: &max, - MinCount: &min, + MaxCount: utils.Int32Ptr(1), + MinCount: utils.Int32Ptr(1), ImageId: &ami, BlockDeviceMappings: []ec2types.BlockDeviceMapping{ { + // attach the snapshot to the instance at launch (a new volume will be created) DeviceName: &deviceName, Ebs: &ec2types.EbsBlockDevice{ - DeleteOnTermination: nil, // ? + DeleteOnTermination: utils.BoolPtr(true), Encrypted: nil, // ? SnapshotId: &snapshot.ID, VolumeSize: nil, // default is snapshot size - VolumeType: ec2types.VolumeTypeGp2, // ? + VolumeType: ec2types.VolumeTypeGp2, // TODO need to decide volume type }, }, }, - InstanceType: ec2types.InstanceTypeT2Large, - MetadataOptions: nil, // ? - SecurityGroups: nil, // use default for now - SubnetId: nil, // use default for now - TagSpecifications: nil, // need to specify tags - UserData: nil, // put launch script here + InstanceType: ec2types.InstanceTypeT2Large, // TODO need to decide instance type + SecurityGroups: nil, // use default for now + SubnetId: nil, // use default for now + TagSpecifications: []ec2types.TagSpecification{ + { + ResourceType: ec2types.ResourceTypeInstance, + Tags: vmclarityTags, + }, + }, + UserData: nil, // TODO put launch script here }, func(options *ec2.Options) { options.Region = snapshot.Region }) @@ -256,61 +302,90 @@ func (c *Client) DeleteSnapshot(snapshot types.Snapshot) error { return nil } -func findRegionByID(regions []types.Region, regionID string) types.Region { - for _, region := range regions { - if strings.Compare(region.ID, regionID) == 0 { - return region +func getInstancesFromDescribeInstancesOutput(result *ec2.DescribeInstancesOutput, excludeTags []*types.Tag, regionID string) []types.Instance { + var ret []types.Instance + + for _, reservation := range result.Reservations { + for _, instance := range reservation.Instances { + if hasExcludeTags(excludeTags, instance.Tags) { + continue + } + ret = append(ret, types.Instance{ + ID: *instance.InstanceId, + Region: regionID, + }) } } - return types.Region{} + return ret } -func getRegionVPCsAndSecurityGroups(region types.Region) (vpcs []string, sgs []string) { - for _, vpc := range region.VPCs { - vpcs = append(vpcs, vpc.ID) - for _, sc := range vpc.SecurityGroups { - sgs = append(sgs, sc.ID) +func findRegionByID(regions []types.Region, regionID string) (types.Region, error) { + for _, region := range regions { + if strings.Compare(region.ID, regionID) == 0 { + return region, nil } } - return + return types.Region{}, fmt.Errorf("region %v not found in regions list", regionID) } -func createDescribeFilters(scopes *types.ScanScope, regionID string) []ec2types.Filter { - var ret []ec2types.Filter - var vpcs []string +func getVPCSecurityGroups(vpc types.VPC) []string { var sgs []string - - if !scopes.All { - region := findRegionByID(scopes.Regions, regionID) - vpcs, sgs = getRegionVPCsAndSecurityGroups(region) + for _, sg := range vpc.SecurityGroups { + sgs = append(sgs, sg.ID) } + return sgs +} - vpcID := "vpc-id" - sgID := "instance.group-id" // TODO +const ( + vpcIDFilterName = "vpc-id" + sgIDFilterName = "instance.group-id" // TODO is this the right one? + instanceStateFilterName = "instance-state-name" +) +func createVPCFilters(vpc types.VPC) []ec2types.Filter { + var ret = make([]ec2types.Filter, 0) + var sgs []string + + // create per vpc filters + ret = append(ret, ec2types.Filter{ + Name: utils.StringPtr(vpcIDFilterName), + Values: []string{vpc.ID}, + }) + sgs = getVPCSecurityGroups(vpc) if len(sgs) > 0 { ret = append(ret, ec2types.Filter{ - Name: &sgID, + Name: utils.StringPtr(sgIDFilterName), Values: sgs, }) - } else if len(vpcs) > 0 { - ret = append(ret, ec2types.Filter{ - Name: &vpcID, - Values: vpcs, + } + + log.Infof("VPC filter created: %+v", ret) + + return ret +} + +func createInstanceStateFilters(scanStopped bool) []ec2types.Filter { + var filters []ec2types.Filter + if scanStopped { + filters = append(filters, ec2types.Filter{ + Name: utils.StringPtr(instanceStateFilterName), + Values: nil, }) } + return filters +} - if len(scopes.IncludeTags) > 0 { - for _, tag := range scopes.IncludeTags { - name := "tag:" + tag.Key - ret = append(ret, ec2types.Filter{ - Name: &name, - Values: []string{tag.Val}, - }) - } +func createInclusionTagsFilters(tags []*types.Tag) []ec2types.Filter { + var filters []ec2types.Filter + + for _, tag := range tags { + filters = append(filters, ec2types.Filter{ + Name: utils.StringPtr("tag:" + tag.Key), + Values: []string{tag.Val}, + }) } - return ret + return filters } func (c *Client) getRegionsToScan(scope *types.ScanScope) ([]string, error) { @@ -328,12 +403,30 @@ func (c *Client) getRegionsToScan(scope *types.ScanScope) ([]string, error) { func (c *Client) ListAllRegions() ([]string, error) { var ret []string - out, err := c.ec2Client.DescribeRegions(context.TODO(), &ec2.DescribeRegionsInput{}) + out, err := c.ec2Client.DescribeRegions(context.TODO(), &ec2.DescribeRegionsInput{ + AllRegions: nil, // display also disabled regions? + }) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to describe regions: %v", err) } for _, region := range out.Regions { ret = append(ret, *region.RegionName) } return ret, nil } + +func hasExcludeTags(excludeTags []*types.Tag, instanceTags []ec2types.Tag) bool { + var excludedTagsMap = make(map[string]string) + + for _, tag := range excludeTags { + excludedTagsMap[tag.Key] = tag.Val + } + for _, tag := range instanceTags { + if val, ok := excludedTagsMap[*tag.Key]; ok { + if strings.Compare(val, *tag.Value) == 0 { + return true + } + } + } + return false +} diff --git a/runtime_scan/pkg/provider/aws/client_test.go b/runtime_scan/pkg/provider/aws/client_test.go index f36ebcd82..65386e237 100644 --- a/runtime_scan/pkg/provider/aws/client_test.go +++ b/runtime_scan/pkg/provider/aws/client_test.go @@ -2,12 +2,16 @@ package aws import ( "context" + "reflect" + "sort" "testing" awsconfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/openclarity/vmclarity/runtime_scan/pkg/types" + "github.com/openclarity/vmclarity/runtime_scan/pkg/utils" ) func TestClient_ListAllRegions(t *testing.T) { @@ -53,3 +57,468 @@ func TestClient_ListAllRegions(t *testing.T) { t.Logf("res: %v", launchedInstance.ID) } + +func Test_createVPCFilters(t *testing.T) { + var ( + vpcID = "vpc-1" + sgID1 = "sg-1" + sgID2 = "sg-2" + + vpcIDFilterName = vpcIDFilterName + sgIDFilterName = sgIDFilterName + ) + + type args struct { + vpc types.VPC + } + tests := []struct { + name string + args args + want []ec2types.Filter + }{ + { + name: "vpc with no security group", + args: args{ + vpc: types.VPC{ + ID: vpcID, + SecurityGroups: nil, + }, + }, + want: []ec2types.Filter{ + { + Name: &vpcIDFilterName, + Values: []string{vpcID}, + }, + }, + }, + { + name: "vpc with one security group", + args: args{ + vpc: types.VPC{ + ID: vpcID, + SecurityGroups: []types.SecurityGroup{ + { + ID: sgID1, + }, + }, + }, + }, + want: []ec2types.Filter{ + { + Name: &vpcIDFilterName, + Values: []string{vpcID}, + }, + { + Name: &sgIDFilterName, + Values: []string{sgID1}, + }, + }, + }, + { + name: "vpc with two security groups", + args: args{ + vpc: types.VPC{ + ID: vpcID, + SecurityGroups: []types.SecurityGroup{ + { + ID: sgID1, + }, + { + ID: sgID2, + }, + }, + }, + }, + want: []ec2types.Filter{ + { + Name: &vpcIDFilterName, + Values: []string{vpcID}, + }, + { + Name: &sgIDFilterName, + Values: []string{sgID1, sgID2}, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := createVPCFilters(tt.args.vpc); !reflect.DeepEqual(got, tt.want) { + t.Errorf("createVPCFilters() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_findRegionByID(t *testing.T) { + type args struct { + regions []types.Region + regionID string + } + tests := []struct { + name string + args args + want types.Region + wantErr bool + }{ + { + name: "found", + args: args{ + regions: []types.Region{ + { + ID: "region-2", + VPCs: nil, + }, + { + ID: "region-1", + VPCs: nil, + }, + }, + regionID: "region-1", + }, + want: types.Region{ + ID: "region-1", + VPCs: nil, + }, + wantErr: false, + }, + { + name: "not found", + args: args{ + regions: []types.Region{ + { + ID: "region-1", + VPCs: nil, + }, + }, + regionID: "region-2", + }, + want: types.Region{}, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := findRegionByID(tt.args.regions, tt.args.regionID) + if (err != nil) != tt.wantErr { + t.Errorf("findRegionByID() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("findRegionByID() got = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_createInclusionTagsFilters(t *testing.T) { + var ( + tagName = "foo" + filterTagName = "tag:" + tagName + tagVal = "bar" + ) + + type args struct { + tags []*types.Tag + } + tests := []struct { + name string + args args + want []ec2types.Filter + }{ + { + name: "no tags", + args: args{ + tags: nil, + }, + want: nil, + }, + { + name: "1 tag", + args: args{ + tags: []*types.Tag{ + { + Key: tagName, + Val: tagVal, + }, + }, + }, + want: []ec2types.Filter{ + { + Name: &filterTagName, + Values: []string{tagVal}, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := createInclusionTagsFilters(tt.args.tags); !reflect.DeepEqual(got, tt.want) { + t.Errorf("createInclusionTagsFilters() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_hasExcludedTags(t *testing.T) { + var ( + tagName1 = "foo1" + tagName2 = "foo2" + tagVal1 = "bar1" + tagVal2 = "bar2" + ) + + type args struct { + excludeTags []*types.Tag + instanceTags []ec2types.Tag + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "instance has no tags", + args: args{ + excludeTags: []*types.Tag{ + { + Key: tagName1, + Val: tagVal1, + }, + { + Key: "stam1", + Val: "stam2", + }, + }, + instanceTags: nil, + }, + want: false, + }, + { + name: "empty excluded tags", + args: args{ + excludeTags: nil, + instanceTags: []ec2types.Tag{ + { + Key: &tagName1, + Value: &tagVal1, + }, + { + Key: &tagName2, + Value: &tagVal2, + }, + }, + }, + want: false, + }, + { + name: "instance has excluded tags", + args: args{ + excludeTags: []*types.Tag{ + { + Key: tagName1, + Val: tagVal1, + }, + { + Key: "stam1", + Val: "stam2", + }, + }, + instanceTags: []ec2types.Tag{ + { + Key: &tagName1, + Value: &tagVal1, + }, + { + Key: &tagName2, + Value: &tagVal2, + }, + }, + }, + want: true, + }, + { + name: "instance does not have excluded tags", + args: args{ + excludeTags: []*types.Tag{ + { + Key: "stam1", + Val: "stam2", + }, + { + Key: "stam3", + Val: "stam4", + }, + }, + instanceTags: []ec2types.Tag{ + { + Key: &tagName1, + Value: &tagVal1, + }, + { + Key: &tagName2, + Value: &tagVal2, + }, + }, + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := hasExcludeTags(tt.args.excludeTags, tt.args.instanceTags); got != tt.want { + t.Errorf("hasExcludeTags() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_getInstancesFromDescribeInstancesOutput(t *testing.T) { + type args struct { + result *ec2.DescribeInstancesOutput + excludeTags []*types.Tag + regionID string + } + tests := []struct { + name string + args args + want []types.Instance + }{ + { + name: "no reservations found", + args: args{ + result: &ec2.DescribeInstancesOutput{ + Reservations: []ec2types.Reservation{}, + }, + excludeTags: nil, + regionID: "region-1", + }, + want: nil, + }, + { + name: "no excluded tags", + args: args{ + result: &ec2.DescribeInstancesOutput{ + Reservations: []ec2types.Reservation{ + { + Instances: []ec2types.Instance{ + { + InstanceId: utils.StringPtr("instance-1"), + Tags: []ec2types.Tag{ + { + Key: utils.StringPtr("key-1"), + Value: utils.StringPtr("val-1"), + }, + }, + }, + { + InstanceId: utils.StringPtr("instance-2"), + Tags: []ec2types.Tag{ + { + Key: utils.StringPtr("key-2"), + Value: utils.StringPtr("val-2"), + }, + }, + }, + }, + }, + { + Instances: []ec2types.Instance{ + { + InstanceId: utils.StringPtr("instance-3"), + }, + }, + }, + }, + }, + excludeTags: nil, + regionID: "region-1", + }, + want: []types.Instance{ + { + ID: "instance-1", + Region: "region-1", + }, + { + ID: "instance-2", + Region: "region-1", + }, + { + ID: "instance-3", + Region: "region-1", + }, + }, + }, + { + name: "one excluded instance", + args: args{ + result: &ec2.DescribeInstancesOutput{ + Reservations: []ec2types.Reservation{ + { + Instances: []ec2types.Instance{ + { + InstanceId: utils.StringPtr("instance-1"), + Tags: []ec2types.Tag{ + { + Key: utils.StringPtr("key-1"), + Value: utils.StringPtr("val-1"), + }, + }, + }, + { + InstanceId: utils.StringPtr("instance-2"), + Tags: []ec2types.Tag{ + { + Key: utils.StringPtr("key-2"), + Value: utils.StringPtr("val-2"), + }, + }, + }, + }, + }, + { + Instances: []ec2types.Instance{ + { + InstanceId: utils.StringPtr("instance-3"), + }, + }, + }, + }, + }, + excludeTags: []*types.Tag{ + { + Key: "key-1", + Val: "val-1", + }, + }, + regionID: "region-1", + }, + want: []types.Instance{ + { + ID: "instance-2", + Region: "region-1", + }, + { + ID: "instance-3", + Region: "region-1", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := getInstancesFromDescribeInstancesOutput(tt.args.result, tt.args.excludeTags, tt.args.regionID) + + sort.Slice(got, func(i, j int) bool { + return got[i].ID > got[j].ID + }) + sort.Slice(tt.want, func(i, j int) bool { + return tt.want[i].ID > tt.want[j].ID + }) + + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("getInstancesFromDescribeInstancesOutput() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/runtime_scan/pkg/provider/provider.go b/runtime_scan/pkg/provider/provider.go index dbe64d5ed..ae399dc5f 100644 --- a/runtime_scan/pkg/provider/provider.go +++ b/runtime_scan/pkg/provider/provider.go @@ -3,22 +3,23 @@ package provider import types2 "github.com/openclarity/vmclarity/runtime_scan/pkg/types" type Client interface { - // list VM instance ids in the account according to the filters. + // Discover - list VM instances in the account according to the filters. Discover(filters *types2.ScanScope) ([]types2.Instance, error) - // create a snapshot of a volume, and return the snapshot id. - CreateSnapshot(types2.Volume) (types2.Snapshot, error) - // copy the snapshot from src region to dest region and return the snapshot id. + // Create a snapshot of a volume. + // return the newly created snapshot. + CreateSnapshot(volume types2.Volume) (types2.Snapshot, error) + // CopySnapshot - copy the snapshot from src region to dest region. + // return the newly created (copy) snapshot. CopySnapshot(snapshot types2.Snapshot, dstRegion string) (types2.Snapshot, error) - // + // WaitForSnapshotReady - wait until snapshot state is 'completed'. WaitForSnapshotReady(snapshot types2.Snapshot) error - // get the instance root volume + // GetInstanceRootVolume - get the instance's root volume. GetInstanceRootVolume(instance types2.Instance) (types2.Volume, error) - // attach a volume to an instance - //AttachVolume(volumeID, instanceID, region string) (string, error) - // + // LaunchInstance - launch an instance. the snapshot will be attached to the instance at launch. + // return the launched instance LaunchInstance(ami, deviceName string, snapshot types2.Snapshot) (types2.Instance, error) - // - DeleteInstance(types2.Instance) error - // - DeleteSnapshot(types2.Snapshot) error + // DeleteInstance - delete an instance. + DeleteInstance(instance types2.Instance) error + // DeleteSnapshot - delete a snapshot. + DeleteSnapshot(snapshot types2.Snapshot) error } diff --git a/runtime_scan/pkg/scanner/job_managment.go b/runtime_scan/pkg/scanner/job_managment.go index fe4e5f082..364c829de 100644 --- a/runtime_scan/pkg/scanner/job_managment.go +++ b/runtime_scan/pkg/scanner/job_managment.go @@ -23,16 +23,16 @@ import ( log "github.com/sirupsen/logrus" "github.com/openclarity/vmclarity/runtime_scan/pkg/config" - types2 "github.com/openclarity/vmclarity/runtime_scan/pkg/types" + "github.com/openclarity/vmclarity/runtime_scan/pkg/types" ) // run jobs. func (s *Scanner) jobBatchManagement(scanDone chan struct{}) { s.Lock() - imageIDToScanData := s.instanceIDToScanData + instanceIDToScanData := s.instanceIDToScanData numberOfWorkers := s.scanConfig.MaxScanParallelism - imagesStartedToScan := &s.progress.ImagesStartedToScan - imagesCompletedToScan := &s.progress.ImagesCompletedToScan + instancesStartedToScan := &s.progress.InstancesStartedToScan + instancesCompletedToScan := &s.progress.InstancesCompletedToScan s.Unlock() // queue of scan data @@ -49,10 +49,10 @@ func (s *Scanner) jobBatchManagement(scanDone chan struct{}) { // wait until scan of all images is done - non blocking. once all done, notify on fullScanDone chan go func() { - for c := 0; c < len(imageIDToScanData); c++ { + for c := 0; c < len(instanceIDToScanData); c++ { select { case <-done: - atomic.AddUint32(imagesCompletedToScan, 1) + atomic.AddUint32(instancesCompletedToScan, 1) case <-s.killSignal: log.WithFields(s.logFields).Debugf("Scan process was canceled - stop waiting for finished jobs") return @@ -63,13 +63,13 @@ func (s *Scanner) jobBatchManagement(scanDone chan struct{}) { }() // send all scan data on scan data queue, for workers to pick it up. - for _, data := range imageIDToScanData { + for _, data := range instanceIDToScanData { go func(data *scanData, ks chan bool) { select { case q <- data: - atomic.AddUint32(imagesStartedToScan, 1) + atomic.AddUint32(instancesStartedToScan, 1) case <-ks: - log.WithFields(s.logFields).Debugf("Scan process was canceled. imageID=%v, scanUUID=%v", data.instance.ID, data.scanUUID) + log.WithFields(s.logFields).Debugf("Scan process was canceled. instanceID=%v, scanUUID=%v", data.instance.ID, data.scanUUID) return } }(data, s.killSignal) @@ -97,10 +97,10 @@ func (s *Scanner) worker(queue chan *scanData, workNumber int, done, ks chan boo log.WithFields(s.logFields).Error(errMsg) s.Lock() data.success = false - data.scanErr = &types2.ScanError{ + data.scanErr = &types.ScanError{ ErrMsg: err.Error(), - ErrType: string(types2.JobRun), - ErrSource: types2.ScanErrSourceJob, + ErrType: string(types.JobRun), + ErrSource: types.ScanErrSourceJob, } data.completed = true s.Unlock() @@ -123,68 +123,69 @@ func (s *Scanner) worker(queue chan *scanData, workNumber int, done, ks chan boo } func (s *Scanner) waitForResult(data *scanData, ks chan bool) { - //log.WithFields(s.logFields).Infof("Waiting for result. imageID=%+v", data.imageID) + log.WithFields(s.logFields).Infof("Waiting for result. instanceID=%+v", data.instance.ID) ticker := time.NewTicker(s.scanConfig.JobResultTimeout) select { case <-data.resultChan: log.WithFields(s.logFields).Infof("Instance scanned result has arrived. instanceID=%v", data.instance.ID) case <-ticker.C: - errMsg := fmt.Errorf("job has timed out. imageID=%v", data.instance.ID) + errMsg := fmt.Errorf("job has timed out. instanceID=%v", data.instance.ID) log.WithFields(s.logFields).Warn(errMsg) s.Lock() data.success = false - data.scanErr = &types2.ScanError{ + data.scanErr = &types.ScanError{ ErrMsg: errMsg.Error(), - ErrType: string(types2.JobTimeout), - ErrSource: types2.ScanErrSourceJob, + ErrType: string(types.JobTimeout), + ErrSource: types.ScanErrSourceJob, } data.timeout = true data.completed = true s.Unlock() case <-ks: - log.WithFields(s.logFields).Infof("Image scan was canceled. imageID=%v", data.instance.ID) + log.WithFields(s.logFields).Infof("Instance scan was canceled. instanceID=%v", data.instance.ID) } } -func (s *Scanner) runJob(data *scanData) (types2.Job, error) { +func (s *Scanner) runJob(data *scanData) (types.Job, error) { rootVolume, err := s.providerClient.GetInstanceRootVolume(data.instance) if err != nil { - return types2.Job{}, fmt.Errorf("failed to get instance root volume. instance id=%v: %v", data.instance.ID, err) + return types.Job{}, fmt.Errorf("failed to get instance root volume. instance id=%v: %v", data.instance.ID, err) } - // create a snapshot of that vm + // create a snapshot of the root volume srcSnapshot, err := s.providerClient.CreateSnapshot(rootVolume) if err != nil { - return types2.Job{}, fmt.Errorf("failed to create snapshot: %v", err) + return types.Job{}, fmt.Errorf("failed to create snapshot: %v", err) } if err := s.providerClient.WaitForSnapshotReady(srcSnapshot); err != nil { - return types2.Job{}, err + return types.Job{}, fmt.Errorf("failed to wait for snapshot to be ready: %v", err) } //copy the snapshot to the scanner region // TODO make sure we need this. + // TODO check if scanner region is same as snapshot region? cpySnapshot, err := s.providerClient.CopySnapshot(srcSnapshot, s.region) if err != nil { - return types2.Job{}, fmt.Errorf("failed to copy snapshot: %v", err) + return types.Job{}, fmt.Errorf("failed to copy snapshot: %v", err) } if err := s.providerClient.WaitForSnapshotReady(cpySnapshot); err != nil { - return types2.Job{}, err + return types.Job{}, fmt.Errorf("failed to wait for snapshot to be ready: %v", err) } // create the scanner job (vm) with a boot script - launchedInstance, err := s.providerClient.LaunchInstance(s.amiID, "", cpySnapshot) + launchedInstance, err := s.providerClient.LaunchInstance(s.jobAMI, "xvdh", cpySnapshot) if err != nil { - return types2.Job{}, fmt.Errorf("failed to launch instance: %v", err) + return types.Job{}, fmt.Errorf("failed to launch instance: %v", err) } - return types2.Job{ + return types.Job{ Instance: launchedInstance, SrcSnapshot: srcSnapshot, DstSnapshot: cpySnapshot, }, nil } -func (s *Scanner) deleteJobIfNeeded(job *types2.Job, isSuccessfulJob, isCompletedJob bool) { +func (s *Scanner) deleteJobIfNeeded(job *types.Job, isSuccessfulJob, isCompletedJob bool) { if job == nil { return } @@ -207,7 +208,7 @@ func (s *Scanner) deleteJobIfNeeded(job *types2.Job, isSuccessfulJob, isComplete } } -func (s *Scanner) deleteJob(job *types2.Job) { +func (s *Scanner) deleteJob(job *types.Job) { if err := s.providerClient.DeleteInstance(job.Instance); err != nil { log.Errorf("failed to delete instance: %v", err) } @@ -218,157 +219,3 @@ func (s *Scanner) deleteJob(job *types2.Job) { log.Errorf("failed to delete dest snapshot: %v", err) } } - -// Due to K8s names constraint we will take the image name w/o the tag and repo. -//func getSimpleImageName(imageName string) (string, error) { -// ref, err := reference.ParseNormalizedNamed(imageName) -// if err != nil { -// return "", fmt.Errorf("failed to parse image name. name=%v: %v", imageName, err) -// } -// -// refName := ref.Name() -// // Take only image name from repo path (ex. solsson/kafka ==> kafka) -// repoEnd := strings.LastIndex(refName, "/") -// -// return refName[repoEnd+1:], nil -//} - -// Job names require their names to follow the DNS label standard as defined in RFC 1123 -// Note: job name is added as a label to the pod template spec so it should follow the DNS label standard and not just DNS-1123 subdomain -// -// This means the name must: -// * contain at most 63 characters -// * contain only lowercase alphanumeric characters or ‘-’ -// * start with an alphanumeric character -// * end with an alphanumeric character. -//func createJobName(imageName string) (string, error) { -// //simpleName, err := getSimpleImageName(imageName) -// //if err != nil { -// // return "", err -// //} -// -// jobName := "scanner-" + simpleName + "-" + uuid.NewV4().String() -// -// // contain at most 63 characters -// jobName = stringsutils.TruncateString(jobName, k8s.MaxK8sJobName) -// -// // contain only lowercase alphanumeric characters or ‘-’ -// jobName = strings.ToLower(jobName) -// jobName = strings.ReplaceAll(jobName, "_", "-") -// -// // no need to validate start, we are using 'jobName' -// -// // end with an alphanumeric character -// jobName = strings.TrimRight(jobName, "-") -// -// return jobName, nil -//} - -//func (s *Scanner) createJob(data *scanData) (*batchv1.Job, error) { -// // We will scan each image once, based on the first pod context. The result will be applied for all other pods with this image. -// podContext := data.contexts[0] -// -// jobName, err := createJobName(podContext.imageName) -// if err != nil { -// return nil, fmt.Errorf("failed to create job name. namespace=%v, pod=%v, container=%v, image=%v, hash=%v: %v", -// podContext.namespace, podContext.podName, podContext.containerName, podContext.imageName, data.imageHash, err) -// } -// -// // Set job values on scanner job template -// job := s.scannerJobTemplate.DeepCopy() -// if !data.shouldScanCISDockerBenchmark { -// removeCISDockerBenchmarkScannerFromJob(job) -// } -// job.SetName(jobName) -// job.SetNamespace(podContext.namespace) -// setJobScanUUID(job, data.scanUUID) -// setJobImageIDToScan(job, data.imageID) -// setJobImageHashToScan(job, data.imageHash) -// setJobImageNameToScan(job, podContext.imageName) -// if podContext.imagePullSecret != "" { -// log.WithFields(s.logFields).Debugf("Adding private registry credentials to image: %s", podContext.imageName) -// setJobDockerConfigFromImagePullSecret(job, podContext.imagePullSecret) -// } else { -// // Use private repo sa credentials only if there is no imagePullSecret -// for _, adder := range s.credentialAdders { -// if adder.ShouldAdd() { -// adder.Add(job) -// } -// } -// } -// -// return job, nil -//} -// -//func removeCISDockerBenchmarkScannerFromJob(job *batchv1.Job) { -// var containers []corev1.Container -// for i := range job.Spec.Template.Spec.Containers { -// container := job.Spec.Template.Spec.Containers[i] -// if container.Name != cisDockerBenchmarkScannerContainerName { -// containers = append(containers, container) -// } -// } -// job.Spec.Template.Spec.Containers = containers -//} - -// Create docker config from imagePullSecret that contains the username and the password required to pull the image. -// We need to do the following: -// 1. Create a volume that holds the `secretName` data -// 2. Mount the volume into each container to a specific path (`BasicVolumeMountPath`/`DockerConfigFileName`) -// 3. Set `DOCKER_CONFIG` to point to the directory that contains the config.json. -//func setJobDockerConfigFromImagePullSecret(job *batchv1.Job, secretName string) { -// job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes, corev1.Volume{ -// Name: _creds.BasicVolumeName, -// VolumeSource: corev1.VolumeSource{ -// Secret: &corev1.SecretVolumeSource{ -// SecretName: secretName, -// Items: []corev1.KeyToPath{ -// { -// Key: corev1.DockerConfigJsonKey, -// Path: _creds.DockerConfigFileName, -// }, -// }, -// }, -// }, -// }) -// for i := range job.Spec.Template.Spec.Containers { -// container := &job.Spec.Template.Spec.Containers[i] -// container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ -// Name: _creds.BasicVolumeName, -// ReadOnly: true, -// MountPath: _creds.BasicVolumeMountPath, -// }) -// container.Env = append(container.Env, corev1.EnvVar{ -// Name: _creds.DockerConfigEnvVar, -// Value: _creds.BasicVolumeMountPath, -// }) -// } -//} - -//func setJobImageIDToScan(job *batchv1.Job, imageID string) { -// for i := range job.Spec.Template.Spec.Containers { -// container := &job.Spec.Template.Spec.Containers[i] -// container.Env = append(container.Env, corev1.EnvVar{Name: shared.ImageIDToScan, Value: imageID}) -// } -//} -// -//func setJobImageHashToScan(job *batchv1.Job, imageHash string) { -// for i := range job.Spec.Template.Spec.Containers { -// container := &job.Spec.Template.Spec.Containers[i] -// container.Env = append(container.Env, corev1.EnvVar{Name: shared.ImageHashToScan, Value: imageHash}) -// } -//} -// -//func setJobImageNameToScan(job *batchv1.Job, imageName string) { -// for i := range job.Spec.Template.Spec.Containers { -// container := &job.Spec.Template.Spec.Containers[i] -// container.Env = append(container.Env, corev1.EnvVar{Name: shared.ImageNameToScan, Value: imageName}) -// } -//} -// -//func setJobScanUUID(job *batchv1.Job, scanUUID string) { -// for i := range job.Spec.Template.Spec.Containers { -// container := &job.Spec.Template.Spec.Containers[i] -// container.Env = append(container.Env, corev1.EnvVar{Name: shared.ScanUUID, Value: scanUUID}) -// } -//} diff --git a/runtime_scan/pkg/scanner/scanner.go b/runtime_scan/pkg/scanner/scanner.go index 6207ccb27..7f6a5356e 100644 --- a/runtime_scan/pkg/scanner/scanner.go +++ b/runtime_scan/pkg/scanner/scanner.go @@ -19,6 +19,7 @@ import ( "fmt" "go/scanner" "sync" + "sync/atomic" uuid "github.com/satori/go.uuid" log "github.com/sirupsen/logrus" @@ -36,31 +37,25 @@ type Scanner struct { progress types.ScanProgress logFields log.Fields - region string - vpcID string - subnetID string - amiID string + region string + jobAMI string sync.Mutex } type scanData struct { - instance types.Instance - //contexts []*imagePodContext // All the pods that contain this image hash + instance types.Instance scanUUID string vulnerabilitiesResult vulnerabilitiesScanResult - //cisDockerBenchmarkResult cisDockerBenchmarkScanResult - //shouldScanCISDockerBenchmark bool - resultChan chan bool - success bool - completed bool - timeout bool - scanErr *types.ScanError + resultChan chan bool + success bool + completed bool + timeout bool + scanErr *types.ScanError } type vulnerabilitiesScanResult struct { - result []string - //layerCommands []*models.ResourceLayerCommand + result []string success bool completed bool error *scanner.Error @@ -75,9 +70,7 @@ func CreateScanner(config *_config.Config, providerClient provider.Client) *Scan progress: types.ScanProgress{}, logFields: nil, region: config.Region, - vpcID: config.VpcID, - subnetID: config.SubnetID, - amiID: config.AmiID, + jobAMI: config.AmiID, Mutex: sync.Mutex{}, } @@ -87,85 +80,27 @@ func CreateScanner(config *_config.Config, providerClient provider.Client) *Scan // initScan Calculate properties of scan targets // nolint:cyclop func (s *Scanner) initScan() error { - //// Get all target pods - //for _, namespace := range s.scanConfig.Instances { - // podList, err := s.clientset.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{}) - // if err != nil { - // return fmt.Errorf("failed to list pods. namespace=%s: %v", namespace, err) - // } - // podsToScan = append(podsToScan, podList.Items...) - // if namespace == corev1.NamespaceAll { - // break - // } - //} - imageIDToScanData := make(map[string]*scanData) - // Populate the image to scanData map from all target pods + // Populate the instance to scanData map for _, instance := range s.scanConfig.Instances { - //if s.shouldIgnorePod(&podsToScan[i]) { - // continue - //} - - // TODO: (idanf) verify if we need to read pod image pull secrets or just mount it to the scanner job - //secrets := k8sutils.GetPodImagePullSecrets(s.clientset, pod) - - // Due to scenarios where image name in the `pod.Status.ContainerStatuses` is different - // from image name in the `pod.Spec.Containers` we will take only image id from `pod.Status.ContainerStatuses`. - //containerNameToImageID := make(map[string]string) - //for _, container := range append(pod.Status.ContainerStatuses, pod.Status.InitContainerStatuses...) { - // containerNameToImageID[container.Name] = k8sutils.NormalizeImageID(container.ImageID) - //} - - //containers := append(pod.Spec.Containers, pod.Spec.InitContainers...) - - //for _, container := range containers { - //imageID, ok := containerNameToImageID[container.Name] - //if !ok { - // log.Warnf("Image id is missing. pod=%v, namepspace=%v, container=%v ,image=%v", - // pod.GetName(), pod.GetNamespace(), container.Name, container.Image) - // continue - //} - //imageHash := k8sutils.ParseImageHash(imageID) - //if imageHash == "" { - // log.WithFields(s.logFields).Warnf("Failed to get image hash - ignoring image. "+ - // "pod=%v, namepspace=%v, image name=%v", pod.GetName(), pod.GetNamespace(), container.Image) - // continue - //} - //// Create pod context - //podContext := &imagePodContext{ - // containerName: container.Name, - // podName: pod.GetName(), - // namespace: pod.GetNamespace(), - // imagePullSecret: k8sutils.GetMatchingSecretName(secrets, container.Image), - // imageName: container.Image, - // podUID: string(pod.GetUID()), - // podLabels: labels.Set(pod.GetLabels()), - //} - //if data, ok := instanceIDToScanData[imageID]; !ok { - // Image added for the first time, create scan data and append pod context - imageIDToScanData[instance.ID] = &scanData{ - instance: instance, - scanUUID: uuid.NewV4().String(), - vulnerabilitiesResult: vulnerabilitiesScanResult{}, - //shouldScanCISDockerBenchmark: s.scanConfig.ShouldScanCISDockerBenchmark, - resultChan: make(chan bool), - success: false, - completed: false, - timeout: false, - scanErr: nil, - } - //} else { - // Image already exist in map, just append the pod context - //data.contexts = append(data.contexts, podContext) - //} + imageIDToScanData[instance.ID] = &scanData{ + instance: instance, + scanUUID: uuid.NewV4().String(), + vulnerabilitiesResult: vulnerabilitiesScanResult{}, + //shouldScanCISDockerBenchmark: s.scanConfig.ShouldScanCISDockerBenchmark, + resultChan: make(chan bool), + success: false, + completed: false, + timeout: false, + scanErr: nil, } - //} + } s.instanceIDToScanData = imageIDToScanData - s.progress.ImagesToScan = uint32(len(imageIDToScanData)) + s.progress.InstancesToScan = uint32(len(imageIDToScanData)) - log.WithFields(s.logFields).Infof("Total %d unique images to scan", s.progress.ImagesToScan) + log.WithFields(s.logFields).Infof("Total %d unique images to scan", s.progress.InstancesToScan) return nil } @@ -185,7 +120,7 @@ func (s *Scanner) Scan(scanConfig *_config.ScanConfig, scanDone chan struct{}) e return fmt.Errorf("failed to initiate scan: %v", err) } - if s.progress.ImagesToScan == 0 { + if s.progress.InstancesToScan == 0 { log.WithFields(s.logFields).Info("Nothing to scan") s.progress.SetStatus(types.NothingToScan) nonBlockingNotification(scanDone) @@ -201,6 +136,46 @@ func (s *Scanner) Scan(scanConfig *_config.ScanConfig, scanDone chan struct{}) e s.Unlock() }() - return nil } + +func (s *Scanner) ScanProgress() types.ScanProgress { + return types.ScanProgress{ + InstancesToScan: s.progress.InstancesToScan, + InstancesStartedToScan: atomic.LoadUint32(&s.progress.InstancesStartedToScan), + InstancesCompletedToScan: atomic.LoadUint32(&s.progress.InstancesCompletedToScan), + Status: s.progress.Status, + } +} + +func (s *Scanner) Results() *types.ScanResults { + s.Lock() + defer s.Unlock() + + var instanceScanResults []*types.InstanceScanResult + + for _, scanD := range s.instanceIDToScanData { + if !scanD.completed { + continue + } + instanceScanResults = append(instanceScanResults, &types.InstanceScanResult{ + Instances: scanD.instance, + Vulnerabilities: scanD.vulnerabilitiesResult.result, + Success: scanD.success, + //ScanErrors: scanD.getScanErrors(), + }) + } + + return &types.ScanResults{ + InstanceScanResults: instanceScanResults, + Progress: s.ScanProgress(), + } +} + +func (s *Scanner) Clear() { + s.Lock() + defer s.Unlock() + + log.WithFields(s.logFields).Infof("Clearing...") + close(s.killSignal) +} diff --git a/runtime_scan/pkg/types/types.go b/runtime_scan/pkg/types/types.go index fded8a56f..f3f28cdda 100644 --- a/runtime_scan/pkg/types/types.go +++ b/runtime_scan/pkg/types/types.go @@ -1,24 +1,5 @@ package types -type Tag struct { - Key string - Val string -} - -type SecurityGroup struct { - ID string -} - -type VPC struct { - ID string - SecurityGroups []SecurityGroup -} - -type Region struct { - ID string - VPCs []VPC -} - type ScanScope struct { All bool Regions []Region @@ -39,27 +20,47 @@ const ( ) type ScanProgress struct { - ImagesToScan uint32 - ImagesStartedToScan uint32 - ImagesCompletedToScan uint32 - Status Status + InstancesToScan uint32 + InstancesStartedToScan uint32 + InstancesCompletedToScan uint32 + Status Status } func (s *ScanProgress) SetStatus(status Status) { s.Status = status } -type VMScanResult struct { - // VM data - VMID string +type InstanceScanResult struct { + // Instance data + Instances Instance // Scan results - Vulnerabilities []string + Vulnerabilities []string // TODO define vulnerabilities struct Success bool + ScanErrors []*ScanError } type ScanResults struct { - ImageScanResults []*VMScanResult - Progress ScanProgress + InstanceScanResults []*InstanceScanResult + Progress ScanProgress +} + +type Tag struct { + Key string + Val string +} + +type SecurityGroup struct { + ID string +} + +type VPC struct { + ID string + SecurityGroups []SecurityGroup +} + +type Region struct { + ID string + VPCs []VPC } type Job struct { diff --git a/runtime_scan/pkg/utils/utils.go b/runtime_scan/pkg/utils/utils.go new file mode 100644 index 000000000..324a9f6a7 --- /dev/null +++ b/runtime_scan/pkg/utils/utils.go @@ -0,0 +1,16 @@ +package utils + +func StringPtr(val string) *string { + ret := val + return &ret +} + +func BoolPtr(val bool) *bool { + ret := val + return &ret +} + +func Int32Ptr(val int32) *int32 { + ret := val + return &ret +} From 7c07e9e100553426fb55ee1de3b7904181ff6389 Mon Sep 17 00:00:00 2001 From: Erez Fishhimer Date: Sun, 13 Nov 2022 10:17:40 +0200 Subject: [PATCH 03/23] discovery + scanning --- runtime_scan/pkg/config/config.go | 21 +- runtime_scan/pkg/config/scan_config.go | 2 +- runtime_scan/pkg/orchestrator/orchestrator.go | 5 +- runtime_scan/pkg/provider/aws/client.go | 101 ++++++--- runtime_scan/pkg/provider/aws/client_test.go | 200 ++++++++---------- runtime_scan/pkg/provider/provider.go | 2 + runtime_scan/pkg/scanner/job_managment.go | 6 +- runtime_scan/pkg/scanner/scanner.go | 33 +-- runtime_scan/pkg/types/types.go | 2 +- 9 files changed, 204 insertions(+), 168 deletions(-) diff --git a/runtime_scan/pkg/config/config.go b/runtime_scan/pkg/config/config.go index ae01af570..5ce911faa 100644 --- a/runtime_scan/pkg/config/config.go +++ b/runtime_scan/pkg/config/config.go @@ -20,19 +20,29 @@ import ( ) const ( - ScannerJobResultListenPort = "SCANNER_JOB_RESULT_LISTEN_PORT" - defaultScannerJobResultListenPort = 8888 + ScannerJobResultListenPort = "SCANNER_JOB_RESULT_LISTEN_PORT" + ScannerRegion = "SCANNER_REGION" + defaultScannerRegion = "us-east-1" + ScannerJobImageID = "SCANNER_JOB_IMAGE_ID" + defaultScannerJobImageID = "ami-0568773882d492fc8" // ubuntu server 22.04 LTS (HVM), SSD volume type + ScannerAttachedVolumeDeviceName = "SCANNER_ATTACHED_VOLUME_DEVICE_NAME" + defaultAttachedVolumeDeviceName = "xvdh" + defaultScannerJobResultListenPort = 8888 ) type Config struct { ScannerJobResultListenPort int - Region string - AmiID string + Region string // scanner region + AmiID string // image id of a scanner job + DeviceName string // the name of the block device to attach to the scanner instance (mounted snapshot) } func setConfigDefaults() { // TODO defaults for region and ami ID viper.SetDefault(ScannerJobResultListenPort, defaultScannerJobResultListenPort) + viper.SetDefault(ScannerRegion, defaultScannerRegion) + viper.SetDefault(ScannerJobImageID, defaultScannerJobImageID) + viper.SetDefault(ScannerAttachedVolumeDeviceName, defaultAttachedVolumeDeviceName) viper.AutomaticEnv() } @@ -42,6 +52,9 @@ func LoadConfig() (*Config, error) { config := &Config{ ScannerJobResultListenPort: viper.GetInt(ScannerJobResultListenPort), + Region: viper.GetString(ScannerRegion), + AmiID: viper.GetString(ScannerJobImageID), + DeviceName: viper.GetString(ScannerAttachedVolumeDeviceName), } return config, nil diff --git a/runtime_scan/pkg/config/scan_config.go b/runtime_scan/pkg/config/scan_config.go index a0c329e41..65dfeb0da 100644 --- a/runtime_scan/pkg/config/scan_config.go +++ b/runtime_scan/pkg/config/scan_config.go @@ -34,7 +34,7 @@ type ScanConfig struct { MaxScanParallelism int // instances to scan Instances []types.Instance - DiscoveryFilters types.ScanScope + ScanScope types.ScanScope JobResultTimeout time.Duration DeleteJobPolicy DeleteJobPolicyType } diff --git a/runtime_scan/pkg/orchestrator/orchestrator.go b/runtime_scan/pkg/orchestrator/orchestrator.go index 0ccb6b0ce..e31426556 100644 --- a/runtime_scan/pkg/orchestrator/orchestrator.go +++ b/runtime_scan/pkg/orchestrator/orchestrator.go @@ -54,7 +54,7 @@ func Create(config *_config.Config) (*Orchestrator, error) { } orc := &Orchestrator{ - scanner: _scanner.CreateScanner(config, awsClient), + scanner: _scanner.CreateScanner(config, awsClient), config: config, providerClient: awsClient, Mutex: sync.Mutex{}, @@ -77,7 +77,7 @@ func (o *Orchestrator) Stop() { } func (o *Orchestrator) Scan(scanConfig *_config.ScanConfig, scanDone chan struct{}) error { - instances, err := o.providerClient.Discover(&scanConfig.DiscoveryFilters) + instances, err := o.providerClient.Discover(&scanConfig.ScanScope) if err != nil { return err } @@ -90,7 +90,6 @@ func (o *Orchestrator) Scan(scanConfig *_config.ScanConfig, scanDone chan struct return nil } - func (o *Orchestrator) ScanProgress() types.ScanProgress { return o.getScanner().ScanProgress() } diff --git a/runtime_scan/pkg/provider/aws/client.go b/runtime_scan/pkg/provider/aws/client.go index 6f822a2bd..00bf17c2e 100644 --- a/runtime_scan/pkg/provider/aws/client.go +++ b/runtime_scan/pkg/provider/aws/client.go @@ -58,16 +58,10 @@ func (c *Client) Discover(scope *types.ScanScope) ([]types.Instance, error) { filters = append(filters, createInclusionTagsFilters(scope.IncludeTags)...) filters = append(filters, createInstanceStateFilters(scope.ScanStopped)...) - for _, regionID := range regions { - region, err := findRegionByID(scope.Regions, regionID) - if err != nil { - log.Errorf("failed to find region by ID: %v", err) - continue - } - + for _, region := range regions { // if no vpcs, that mean that we don't need any vpc filters if len(region.VPCs) == 0 { - instances, err := c.GetInstances(filters, scope.ExcludeTags, regionID) + instances, err := c.GetInstances(filters, scope.ExcludeTags, region.ID) if err != nil { return nil, fmt.Errorf("failed to get instances: %v", err) } @@ -77,9 +71,9 @@ func (c *Client) Discover(scope *types.ScanScope) ([]types.Instance, error) { // need to do a per vpc call for DescribeInstances for _, vpc := range region.VPCs { - filters = append(filters, createVPCFilters(vpc)...) + vpcFilters := append(filters, createVPCFilters(vpc)...) - instances, err := c.GetInstances(filters, scope.ExcludeTags, regionID) + instances, err := c.GetInstances(vpcFilters, scope.ExcludeTags, region.ID) if err != nil { return nil, fmt.Errorf("failed to get instances: %v", err) } @@ -158,7 +152,7 @@ func (c *Client) WaitForSnapshotReady(snapshot types.Snapshot) error { options.Region = snapshot.Region }) if err != nil { - return fmt.Errorf("failed to describe snapshot: %v", err) + return fmt.Errorf("failed to describe snapshot. snapshotID=%v: %v", snapshot.ID, err) } if len(out.Snapshots) != 1 { return fmt.Errorf("got unexcpected number of snapshots (%v) with snapshot id %v. excpecting 1", len(out.Snapshots), snapshot.ID) @@ -172,6 +166,32 @@ func (c *Client) WaitForSnapshotReady(snapshot types.Snapshot) error { } } +func (c *Client) WaitForInstanceReady(instance types.Instance) error { + ticker := time.NewTicker(3 * time.Second) + defer ticker.Stop() + timeout := time.After(3 * time.Minute) + + for { + select { + case <-ticker.C: + out, err := c.ec2Client.DescribeInstances(context.TODO(), &ec2.DescribeInstancesInput{ + InstanceIds: []string{instance.ID}, + }, func(options *ec2.Options) { + options.Region = instance.Region + }) + if err != nil { + return fmt.Errorf("failed to describe instance. instanceID=%v: %v", instance.ID, err) + } + state := getInstanceState(out, instance.ID) + if state == ec2types.InstanceStateNameRunning { + return nil + } + case <-timeout: + return fmt.Errorf("timeout") + } + } +} + func (c *Client) CopySnapshot(snapshot types.Snapshot, dstRegion string) (types.Snapshot, error) { snap, err := c.ec2Client.CopySnapshot(context.TODO(), &ec2.CopySnapshotInput{ SourceRegion: &snapshot.Region, @@ -302,6 +322,19 @@ func (c *Client) DeleteSnapshot(snapshot types.Snapshot) error { return nil } +func getInstanceState(result *ec2.DescribeInstancesOutput, instanceID string) ec2types.InstanceStateName { + for _, reservation := range result.Reservations { + for _, instance := range reservation.Instances { + if strings.Compare(*instance.InstanceId, instanceID) == 0 { + if instance.State != nil { + return instance.State.Name + } + } + } + } + return ec2types.InstanceStateNamePending +} + func getInstancesFromDescribeInstancesOutput(result *ec2.DescribeInstancesOutput, excludeTags []*types.Tag, regionID string) []types.Instance { var ret []types.Instance @@ -319,16 +352,7 @@ func getInstancesFromDescribeInstancesOutput(result *ec2.DescribeInstancesOutput return ret } -func findRegionByID(regions []types.Region, regionID string) (types.Region, error) { - for _, region := range regions { - if strings.Compare(region.ID, regionID) == 0 { - return region, nil - } - } - return types.Region{}, fmt.Errorf("region %v not found in regions list", regionID) -} - -func getVPCSecurityGroups(vpc types.VPC) []string { +func getVPCSecurityGroupsIDs(vpc types.VPC) []string { var sgs []string for _, sg := range vpc.SecurityGroups { sgs = append(sgs, sg.ID) @@ -351,7 +375,7 @@ func createVPCFilters(vpc types.VPC) []ec2types.Filter { Name: utils.StringPtr(vpcIDFilterName), Values: []string{vpc.ID}, }) - sgs = getVPCSecurityGroups(vpc) + sgs = getVPCSecurityGroupsIDs(vpc) if len(sgs) > 0 { ret = append(ret, ec2types.Filter{ Name: utils.StringPtr(sgIDFilterName), @@ -366,12 +390,17 @@ func createVPCFilters(vpc types.VPC) []ec2types.Filter { func createInstanceStateFilters(scanStopped bool) []ec2types.Filter { var filters []ec2types.Filter + var states = []string{"running"} if scanStopped { - filters = append(filters, ec2types.Filter{ - Name: utils.StringPtr(instanceStateFilterName), - Values: nil, - }) + states = append(states, "stopped") } + + // TODO these are the states: pending | running | shutting-down | terminated | stopping | stopped + // Do we want to scan any other state (other than running and stopped) + filters = append(filters, ec2types.Filter{ + Name: utils.StringPtr(instanceStateFilterName), + Values: states, + }) return filters } @@ -388,21 +417,21 @@ func createInclusionTagsFilters(tags []*types.Tag) []ec2types.Filter { return filters } -func (c *Client) getRegionsToScan(scope *types.ScanScope) ([]string, error) { +func (c *Client) getRegionsToScan(scope *types.ScanScope) ([]types.Region, error) { if scope.All { return c.ListAllRegions() } - var ret []string + var ret []types.Region for _, region := range scope.Regions { - ret = append(ret, region.ID) + ret = append(ret, region) } return ret, nil } -func (c *Client) ListAllRegions() ([]string, error) { - var ret []string +func (c *Client) ListAllRegions() ([]types.Region, error) { + var ret []types.Region out, err := c.ec2Client.DescribeRegions(context.TODO(), &ec2.DescribeRegionsInput{ AllRegions: nil, // display also disabled regions? }) @@ -410,7 +439,9 @@ func (c *Client) ListAllRegions() ([]string, error) { return nil, fmt.Errorf("failed to describe regions: %v", err) } for _, region := range out.Regions { - ret = append(ret, *region.RegionName) + ret = append(ret, types.Region{ + ID: *region.RegionName, + }) } return ret, nil } @@ -421,9 +452,9 @@ func hasExcludeTags(excludeTags []*types.Tag, instanceTags []ec2types.Tag) bool for _, tag := range excludeTags { excludedTagsMap[tag.Key] = tag.Val } - for _, tag := range instanceTags { - if val, ok := excludedTagsMap[*tag.Key]; ok { - if strings.Compare(val, *tag.Value) == 0 { + for _, instanceTag := range instanceTags { + if val, ok := excludedTagsMap[*instanceTag.Key]; ok { + if strings.Compare(val, *instanceTag.Value) == 0 { return true } } diff --git a/runtime_scan/pkg/provider/aws/client_test.go b/runtime_scan/pkg/provider/aws/client_test.go index 65386e237..4b1f7ff97 100644 --- a/runtime_scan/pkg/provider/aws/client_test.go +++ b/runtime_scan/pkg/provider/aws/client_test.go @@ -1,12 +1,10 @@ package aws import ( - "context" "reflect" "sort" "testing" - awsconfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/ec2" ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" @@ -14,49 +12,100 @@ import ( "github.com/openclarity/vmclarity/runtime_scan/pkg/utils" ) -func TestClient_ListAllRegions(t *testing.T) { - cfg, err := awsconfig.LoadDefaultConfig(context.TODO()) - if err != nil { - t.Fatalf("%v", err) - } - ec2Client := ec2.NewFromConfig(cfg) - - c := Client{ - ec2Client: ec2Client, - } - instance := types.Instance{ - ID: "i-0f70b335ea12b2853", - Region: "us-east-1", - } - rootVolume, err := c.GetInstanceRootVolume(instance) - if err != nil { - t.Fatalf("%v", err) - } - // create a snapshot of that vm - srcSnapshot, err := c.CreateSnapshot(rootVolume) - if err != nil { - t.Fatalf("%v", err) - } - if err := c.WaitForSnapshotReady(srcSnapshot); err != nil { - t.Fatalf("%v", err) - } - //copy the snapshot to the scanner region - // TODO make sure we need this. - cpySnapshot, err := c.CopySnapshot(srcSnapshot, "us-east-2") - if err != nil { - t.Fatalf("%v", err) - } - if err := c.WaitForSnapshotReady(cpySnapshot); err != nil { - t.Fatalf("%v", err) - } - // create the scanner job (vm) with a boot script - launchedInstance, err := c.LaunchInstance("ami-0568773882d492fc8", "xvdh", cpySnapshot) - if err != nil { - t.Fatalf("%v", err) - } - - t.Logf("res: %v", launchedInstance.ID) -} +// +//func TestClient_ListAllRegions(t *testing.T) { +// //cfg, err := awsconfig.LoadDefaultConfig(context.TODO()) +// //if err != nil { +// // t.Fatalf("%v", err) +// //} +// //ec2Client := ec2.NewFromConfig(cfg) +// +// c, err := Create() +// if err != nil { +// t.Fatalf("%v", err) +// } +// //instance := types.Instance { +// // ID: "i-0f70b335ea12b2853", +// // Region: "us-east-1", +// //} +// scope := types.ScanScope{ +// All: false, +// Regions: []types.Region{ +// { +// ID: "us-east-2", +// VPCs: []types.VPC{ +// { +// ID: "vpc-32ea7c59", +// SecurityGroups: []types.SecurityGroup{ +// { +// ID: "sg-4d6b853a", +// }, +// }, +// }, +// }, +// }, +// { +// ID: "us-east-1", +// VPCs: []types.VPC{ +// { +// ID: "vpc-0c41450ba658eed00", +// SecurityGroups: nil, +// }, +// { +// ID: "vpc-9ca32ce1", +// SecurityGroups: []types.SecurityGroup{ +// { +// ID: "sg-030918e8e73254d42", +// }, +// }, +// }, +// }, +// }, +// }, +// ScanStopped: true, +// IncludeTags: []*types.Tag{ +// { +// Key: "Name", +// Val: "diff-vpc", +// }, +// }, +// ExcludeTags: nil, +// } +// instances, err := c.Discover(&scope) +// if err != nil { +// t.Fatalf("%v", err) +// } +// +// instance := instances[0] +// +// rootVolume, err := c.GetInstanceRootVolume(instance) +// if err != nil { +// t.Fatalf("%v", err) +// } +// // create a snapshot of that vm +// srcSnapshot, err := c.CreateSnapshot(rootVolume) +// if err != nil { +// t.Fatalf("%v", err) +// } +// if err := c.WaitForSnapshotReady(srcSnapshot); err != nil { +// t.Fatalf("%v", err) +// } +// //copy the snapshot to the scanner region +// cpySnapshot, err := c.CopySnapshot(srcSnapshot, "us-east-2") +// if err != nil { +// t.Fatalf("%v", err) +// } +// if err := c.WaitForSnapshotReady(cpySnapshot); err != nil { +// t.Fatalf("%v", err) +// } +// // create the scanner job (vm) with a boot script +// launchedInstance, err := c.LaunchInstance("ami-0568773882d492fc8", "xvdh", cpySnapshot) +// if err != nil { +// t.Fatalf("%v", err) +// } +// +// t.Logf("res: %v", launchedInstance.ID) +//} func Test_createVPCFilters(t *testing.T) { var ( @@ -150,67 +199,6 @@ func Test_createVPCFilters(t *testing.T) { } } -func Test_findRegionByID(t *testing.T) { - type args struct { - regions []types.Region - regionID string - } - tests := []struct { - name string - args args - want types.Region - wantErr bool - }{ - { - name: "found", - args: args{ - regions: []types.Region{ - { - ID: "region-2", - VPCs: nil, - }, - { - ID: "region-1", - VPCs: nil, - }, - }, - regionID: "region-1", - }, - want: types.Region{ - ID: "region-1", - VPCs: nil, - }, - wantErr: false, - }, - { - name: "not found", - args: args{ - regions: []types.Region{ - { - ID: "region-1", - VPCs: nil, - }, - }, - regionID: "region-2", - }, - want: types.Region{}, - wantErr: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := findRegionByID(tt.args.regions, tt.args.regionID) - if (err != nil) != tt.wantErr { - t.Errorf("findRegionByID() error = %v, wantErr %v", err, tt.wantErr) - return - } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("findRegionByID() got = %v, want %v", got, tt.want) - } - }) - } -} - func Test_createInclusionTagsFilters(t *testing.T) { var ( tagName = "foo" diff --git a/runtime_scan/pkg/provider/provider.go b/runtime_scan/pkg/provider/provider.go index ae399dc5f..a9e39fabd 100644 --- a/runtime_scan/pkg/provider/provider.go +++ b/runtime_scan/pkg/provider/provider.go @@ -13,6 +13,8 @@ type Client interface { CopySnapshot(snapshot types2.Snapshot, dstRegion string) (types2.Snapshot, error) // WaitForSnapshotReady - wait until snapshot state is 'completed'. WaitForSnapshotReady(snapshot types2.Snapshot) error + // WaitForSnapshotReady - wait until instance state is 'running'. + WaitForInstanceReady(instance types2.Instance) error // GetInstanceRootVolume - get the instance's root volume. GetInstanceRootVolume(instance types2.Instance) (types2.Volume, error) // LaunchInstance - launch an instance. the snapshot will be attached to the instance at launch. diff --git a/runtime_scan/pkg/scanner/job_managment.go b/runtime_scan/pkg/scanner/job_managment.go index 364c829de..32e7415d6 100644 --- a/runtime_scan/pkg/scanner/job_managment.go +++ b/runtime_scan/pkg/scanner/job_managment.go @@ -162,7 +162,6 @@ func (s *Scanner) runJob(data *scanData) (types.Job, error) { } //copy the snapshot to the scanner region - // TODO make sure we need this. // TODO check if scanner region is same as snapshot region? cpySnapshot, err := s.providerClient.CopySnapshot(srcSnapshot, s.region) if err != nil { @@ -173,10 +172,13 @@ func (s *Scanner) runJob(data *scanData) (types.Job, error) { } // create the scanner job (vm) with a boot script - launchedInstance, err := s.providerClient.LaunchInstance(s.jobAMI, "xvdh", cpySnapshot) + launchedInstance, err := s.providerClient.LaunchInstance(s.jobAMI, s.deviceName, cpySnapshot) if err != nil { return types.Job{}, fmt.Errorf("failed to launch instance: %v", err) } + if err := s.providerClient.WaitForInstanceReady(launchedInstance); err != nil { + return types.Job{}, fmt.Errorf("failed to wait for instance to be ready: %v", err) + } return types.Job{ Instance: launchedInstance, diff --git a/runtime_scan/pkg/scanner/scanner.go b/runtime_scan/pkg/scanner/scanner.go index 7f6a5356e..af38fd584 100644 --- a/runtime_scan/pkg/scanner/scanner.go +++ b/runtime_scan/pkg/scanner/scanner.go @@ -31,14 +31,15 @@ import ( type Scanner struct { instanceIDToScanData map[string]*scanData - providerClient provider.Client + progress types.ScanProgress scanConfig *_config.ScanConfig killSignal chan bool - progress types.ScanProgress + providerClient provider.Client logFields log.Fields - region string - jobAMI string + region string + jobAMI string + deviceName string sync.Mutex } @@ -63,14 +64,15 @@ type vulnerabilitiesScanResult struct { func CreateScanner(config *_config.Config, providerClient provider.Client) *Scanner { s := &Scanner{ - instanceIDToScanData: nil, + progress: types.ScanProgress{ + Status: types.Idle, + }, + killSignal: make(chan bool), providerClient: providerClient, - scanConfig: nil, - killSignal: nil, - progress: types.ScanProgress{}, - logFields: nil, + logFields: log.Fields{"scanner id": uuid.NewV4().String()}, region: config.Region, jobAMI: config.AmiID, + deviceName: config.DeviceName, Mutex: sync.Mutex{}, } @@ -88,12 +90,11 @@ func (s *Scanner) initScan() error { instance: instance, scanUUID: uuid.NewV4().String(), vulnerabilitiesResult: vulnerabilitiesScanResult{}, - //shouldScanCISDockerBenchmark: s.scanConfig.ShouldScanCISDockerBenchmark, - resultChan: make(chan bool), - success: false, - completed: false, - timeout: false, - scanErr: nil, + resultChan: make(chan bool), + success: false, + completed: false, + timeout: false, + scanErr: nil, } } @@ -159,7 +160,7 @@ func (s *Scanner) Results() *types.ScanResults { continue } instanceScanResults = append(instanceScanResults, &types.InstanceScanResult{ - Instances: scanD.instance, + Instance: scanD.instance, Vulnerabilities: scanD.vulnerabilitiesResult.result, Success: scanD.success, //ScanErrors: scanD.getScanErrors(), diff --git a/runtime_scan/pkg/types/types.go b/runtime_scan/pkg/types/types.go index f3f28cdda..09fb619f4 100644 --- a/runtime_scan/pkg/types/types.go +++ b/runtime_scan/pkg/types/types.go @@ -32,7 +32,7 @@ func (s *ScanProgress) SetStatus(status Status) { type InstanceScanResult struct { // Instance data - Instances Instance + Instance Instance // Scan results Vulnerabilities []string // TODO define vulnerabilities struct Success bool From 1812b5ad0f5c7ddc12df82d2565972bc323a2851 Mon Sep 17 00:00:00 2001 From: Erez Fishhimer Date: Sun, 13 Nov 2022 11:06:56 +0200 Subject: [PATCH 04/23] small changes --- runtime_scan/pkg/provider/aws/client.go | 2 +- runtime_scan/pkg/scanner/job_managment.go | 4 ++-- runtime_scan/pkg/scanner/scanner.go | 10 +++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/runtime_scan/pkg/provider/aws/client.go b/runtime_scan/pkg/provider/aws/client.go index 00bf17c2e..4f9c02006 100644 --- a/runtime_scan/pkg/provider/aws/client.go +++ b/runtime_scan/pkg/provider/aws/client.go @@ -440,7 +440,7 @@ func (c *Client) ListAllRegions() ([]types.Region, error) { } for _, region := range out.Regions { ret = append(ret, types.Region{ - ID: *region.RegionName, + ID: *region.RegionName, }) } return ret, nil diff --git a/runtime_scan/pkg/scanner/job_managment.go b/runtime_scan/pkg/scanner/job_managment.go index 32e7415d6..bf1a96baf 100644 --- a/runtime_scan/pkg/scanner/job_managment.go +++ b/runtime_scan/pkg/scanner/job_managment.go @@ -47,7 +47,7 @@ func (s *Scanner) jobBatchManagement(scanDone chan struct{}) { go s.worker(q, i, done, s.killSignal) } - // wait until scan of all images is done - non blocking. once all done, notify on fullScanDone chan + // wait until scan of all instances is done - non blocking. once all done, notify on fullScanDone chan go func() { for c := 0; c < len(instanceIDToScanData); c++ { select { @@ -113,7 +113,7 @@ func (s *Scanner) worker(queue chan *scanData, workNumber int, done, ks chan boo select { case done <- true: case <-ks: - log.WithFields(s.logFields).Infof("Image scan was canceled. imageID=%v", data.instance.ID) + log.WithFields(s.logFields).Infof("Instance scan was canceled. instanceID=%v", data.instance.ID) } case <-ks: log.WithFields(s.logFields).Debugf("worker #%v halted", workNumber) diff --git a/runtime_scan/pkg/scanner/scanner.go b/runtime_scan/pkg/scanner/scanner.go index af38fd584..f00f621ae 100644 --- a/runtime_scan/pkg/scanner/scanner.go +++ b/runtime_scan/pkg/scanner/scanner.go @@ -82,11 +82,11 @@ func CreateScanner(config *_config.Config, providerClient provider.Client) *Scan // initScan Calculate properties of scan targets // nolint:cyclop func (s *Scanner) initScan() error { - imageIDToScanData := make(map[string]*scanData) + instanceIDToScanData := make(map[string]*scanData) // Populate the instance to scanData map for _, instance := range s.scanConfig.Instances { - imageIDToScanData[instance.ID] = &scanData{ + instanceIDToScanData[instance.ID] = &scanData{ instance: instance, scanUUID: uuid.NewV4().String(), vulnerabilitiesResult: vulnerabilitiesScanResult{}, @@ -98,10 +98,10 @@ func (s *Scanner) initScan() error { } } - s.instanceIDToScanData = imageIDToScanData - s.progress.InstancesToScan = uint32(len(imageIDToScanData)) + s.instanceIDToScanData = instanceIDToScanData + s.progress.InstancesToScan = uint32(len(instanceIDToScanData)) - log.WithFields(s.logFields).Infof("Total %d unique images to scan", s.progress.InstancesToScan) + log.WithFields(s.logFields).Infof("Total %d unique instances to scan", s.progress.InstancesToScan) return nil } From a96f3a2a499e14088a429351129c1b6761442432 Mon Sep 17 00:00:00 2001 From: Erez Fishhimer Date: Sun, 13 Nov 2022 11:13:52 +0200 Subject: [PATCH 05/23] add licenses --- runtime_scan/pkg/provider/aws/client.go | 15 +++++++++++++++ runtime_scan/pkg/provider/aws/client_test.go | 15 +++++++++++++++ runtime_scan/pkg/provider/provider.go | 15 +++++++++++++++ runtime_scan/pkg/types/types.go | 15 +++++++++++++++ runtime_scan/pkg/utils/utils.go | 15 +++++++++++++++ 5 files changed, 75 insertions(+) diff --git a/runtime_scan/pkg/provider/aws/client.go b/runtime_scan/pkg/provider/aws/client.go index 4f9c02006..daffcf5d2 100644 --- a/runtime_scan/pkg/provider/aws/client.go +++ b/runtime_scan/pkg/provider/aws/client.go @@ -1,3 +1,18 @@ +// Copyright © 2022 Cisco Systems, Inc. and its affiliates. +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package aws import ( diff --git a/runtime_scan/pkg/provider/aws/client_test.go b/runtime_scan/pkg/provider/aws/client_test.go index 4b1f7ff97..2df680a96 100644 --- a/runtime_scan/pkg/provider/aws/client_test.go +++ b/runtime_scan/pkg/provider/aws/client_test.go @@ -1,3 +1,18 @@ +// Copyright © 2022 Cisco Systems, Inc. and its affiliates. +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package aws import ( diff --git a/runtime_scan/pkg/provider/provider.go b/runtime_scan/pkg/provider/provider.go index a9e39fabd..55084e9d3 100644 --- a/runtime_scan/pkg/provider/provider.go +++ b/runtime_scan/pkg/provider/provider.go @@ -1,3 +1,18 @@ +// Copyright © 2022 Cisco Systems, Inc. and its affiliates. +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package provider import types2 "github.com/openclarity/vmclarity/runtime_scan/pkg/types" diff --git a/runtime_scan/pkg/types/types.go b/runtime_scan/pkg/types/types.go index 09fb619f4..19eb2fde3 100644 --- a/runtime_scan/pkg/types/types.go +++ b/runtime_scan/pkg/types/types.go @@ -1,3 +1,18 @@ +// Copyright © 2022 Cisco Systems, Inc. and its affiliates. +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package types type ScanScope struct { diff --git a/runtime_scan/pkg/utils/utils.go b/runtime_scan/pkg/utils/utils.go index 324a9f6a7..eef232875 100644 --- a/runtime_scan/pkg/utils/utils.go +++ b/runtime_scan/pkg/utils/utils.go @@ -1,3 +1,18 @@ +// Copyright © 2022 Cisco Systems, Inc. and its affiliates. +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package utils func StringPtr(val string) *string { From a99ee238fc9d658c8eca94013b7eaad15da2274d Mon Sep 17 00:00:00 2001 From: Erez Fishhimer Date: Sun, 13 Nov 2022 11:34:42 +0200 Subject: [PATCH 06/23] edit makefile and dockerfile --- Dockerfile | 9 +++++++++ Makefile | 8 +++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 36d0288f0..cfafca028 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,6 +8,15 @@ ARG VERSION ARG BUILD_TIMESTAMP ARG COMMIT_HASH +# Copy runtime_scan go.mod & go.sum +WORKDIR /build/runtime_scan +COPY runtime_scan/go.* ./ +RUN go mod download + +# Copy runtime_scan code +WORKDIR /build +COPY runtime_scan ./runtime_scan + # Copy and build backend code COPY . . RUN go build -ldflags="-s -w \ diff --git a/Makefile b/Makefile index c3e67d9fe..84879f8d5 100644 --- a/Makefile +++ b/Makefile @@ -43,6 +43,7 @@ push-docker: docker ## Build and Push VMClarity docker image .PHONY: test test: ## Run Unit Tests @(CGO_ENABLED=0 go test ./...) + @(cd runtime_scan && go test ./...) .PHONY: clean clean: ## Clean all build artifacts @@ -57,10 +58,12 @@ bin/golangci-lint-${GOLANGCI_VERSION}: .PHONY: lint lint: bin/golangci-lint ## Run linter ./bin/golangci-lint run + cd runtime_scan && ../bin/golangci-lint run .PHONY: fix fix: bin/golangci-lint ## Fix lint violations ./bin/golangci-lint run --fix + cd runtime_scan && ../bin/golangci-lint run --fix bin/licensei: bin/licensei-${LICENSEI_VERSION} @ln -sf licensei-${LICENSEI_VERSION} bin/licensei @@ -72,7 +75,6 @@ bin/licensei-${LICENSEI_VERSION}: .PHONY: license-check license-check: bin/licensei ## Run license check ./bin/licensei header - ./bin/licensei check --config=.licensei.toml .PHONY: license-cache license-cache: bin/licensei ## Generate license cache @@ -80,3 +82,7 @@ license-cache: bin/licensei ## Generate license cache .PHONY: check check: lint test ## Run tests and linters + +.PHONY: gomod-tidy +gomod-tidy: + cd runtime_scan && go mod tidy \ No newline at end of file From 78c902d5a47203fc5a82266b31408d19f113ddfe Mon Sep 17 00:00:00 2001 From: Erez Fishhimer Date: Sun, 13 Nov 2022 11:51:02 +0200 Subject: [PATCH 07/23] remove comment --- runtime_scan/pkg/config/config.go | 1 - 1 file changed, 1 deletion(-) diff --git a/runtime_scan/pkg/config/config.go b/runtime_scan/pkg/config/config.go index 5ce911faa..2432599b1 100644 --- a/runtime_scan/pkg/config/config.go +++ b/runtime_scan/pkg/config/config.go @@ -38,7 +38,6 @@ type Config struct { } func setConfigDefaults() { - // TODO defaults for region and ami ID viper.SetDefault(ScannerJobResultListenPort, defaultScannerJobResultListenPort) viper.SetDefault(ScannerRegion, defaultScannerRegion) viper.SetDefault(ScannerJobImageID, defaultScannerJobImageID) From da397986e8e83cb3a89171b03983aa09b0a8154e Mon Sep 17 00:00:00 2001 From: Erez Fishhimer Date: Sun, 13 Nov 2022 12:22:57 +0200 Subject: [PATCH 08/23] use scanner subnet id --- runtime_scan/pkg/config/config.go | 1 + runtime_scan/pkg/provider/aws/client.go | 4 ++-- runtime_scan/pkg/provider/provider.go | 2 +- runtime_scan/pkg/scanner/job_managment.go | 2 +- runtime_scan/pkg/scanner/scanner.go | 16 +++++++++------- 5 files changed, 14 insertions(+), 11 deletions(-) diff --git a/runtime_scan/pkg/config/config.go b/runtime_scan/pkg/config/config.go index 2432599b1..019070650 100644 --- a/runtime_scan/pkg/config/config.go +++ b/runtime_scan/pkg/config/config.go @@ -35,6 +35,7 @@ type Config struct { Region string // scanner region AmiID string // image id of a scanner job DeviceName string // the name of the block device to attach to the scanner instance (mounted snapshot) + SubnetID string // the scanner's subnet ID } func setConfigDefaults() { diff --git a/runtime_scan/pkg/provider/aws/client.go b/runtime_scan/pkg/provider/aws/client.go index daffcf5d2..bb0dfde0f 100644 --- a/runtime_scan/pkg/provider/aws/client.go +++ b/runtime_scan/pkg/provider/aws/client.go @@ -270,7 +270,7 @@ func (c *Client) GetInstanceRootVolume(instance types.Instance) (types.Volume, e return types.Volume{}, fmt.Errorf("failed to find root device volume") } -func (c *Client) LaunchInstance(ami, deviceName string, snapshot types.Snapshot) (types.Instance, error) { +func (c *Client) LaunchInstance(ami, deviceName, subnetID string, snapshot types.Snapshot) (types.Instance, error) { out, err := c.ec2Client.RunInstances(context.TODO(), &ec2.RunInstancesInput{ MaxCount: utils.Int32Ptr(1), MinCount: utils.Int32Ptr(1), @@ -290,7 +290,7 @@ func (c *Client) LaunchInstance(ami, deviceName string, snapshot types.Snapshot) }, InstanceType: ec2types.InstanceTypeT2Large, // TODO need to decide instance type SecurityGroups: nil, // use default for now - SubnetId: nil, // use default for now + SubnetId: &subnetID, TagSpecifications: []ec2types.TagSpecification{ { ResourceType: ec2types.ResourceTypeInstance, diff --git a/runtime_scan/pkg/provider/provider.go b/runtime_scan/pkg/provider/provider.go index 55084e9d3..046d391e4 100644 --- a/runtime_scan/pkg/provider/provider.go +++ b/runtime_scan/pkg/provider/provider.go @@ -34,7 +34,7 @@ type Client interface { GetInstanceRootVolume(instance types2.Instance) (types2.Volume, error) // LaunchInstance - launch an instance. the snapshot will be attached to the instance at launch. // return the launched instance - LaunchInstance(ami, deviceName string, snapshot types2.Snapshot) (types2.Instance, error) + LaunchInstance(ami, deviceName, subnetID string, snapshot types2.Snapshot) (types2.Instance, error) // DeleteInstance - delete an instance. DeleteInstance(instance types2.Instance) error // DeleteSnapshot - delete a snapshot. diff --git a/runtime_scan/pkg/scanner/job_managment.go b/runtime_scan/pkg/scanner/job_managment.go index bf1a96baf..09cba61bd 100644 --- a/runtime_scan/pkg/scanner/job_managment.go +++ b/runtime_scan/pkg/scanner/job_managment.go @@ -172,7 +172,7 @@ func (s *Scanner) runJob(data *scanData) (types.Job, error) { } // create the scanner job (vm) with a boot script - launchedInstance, err := s.providerClient.LaunchInstance(s.jobAMI, s.deviceName, cpySnapshot) + launchedInstance, err := s.providerClient.LaunchInstance(s.jobAMI, s.deviceName, s.subnetID, cpySnapshot) if err != nil { return types.Job{}, fmt.Errorf("failed to launch instance: %v", err) } diff --git a/runtime_scan/pkg/scanner/scanner.go b/runtime_scan/pkg/scanner/scanner.go index f00f621ae..e738a10d0 100644 --- a/runtime_scan/pkg/scanner/scanner.go +++ b/runtime_scan/pkg/scanner/scanner.go @@ -40,6 +40,7 @@ type Scanner struct { region string jobAMI string deviceName string + subnetID string sync.Mutex } @@ -67,13 +68,14 @@ func CreateScanner(config *_config.Config, providerClient provider.Client) *Scan progress: types.ScanProgress{ Status: types.Idle, }, - killSignal: make(chan bool), - providerClient: providerClient, - logFields: log.Fields{"scanner id": uuid.NewV4().String()}, - region: config.Region, - jobAMI: config.AmiID, - deviceName: config.DeviceName, - Mutex: sync.Mutex{}, + killSignal: make(chan bool), + providerClient: providerClient, + logFields: log.Fields{"scanner id": uuid.NewV4().String()}, + region: config.Region, + jobAMI: config.AmiID, + deviceName: config.DeviceName, + subnetID: config.SubnetID, + Mutex: sync.Mutex{}, } return s From 5fbb089a7a90f5a1c38e302b55f028c1738d3ea5 Mon Sep 17 00:00:00 2001 From: Erez Fishhimer Date: Sun, 13 Nov 2022 12:30:25 +0200 Subject: [PATCH 09/23] add some more unit tests --- runtime_scan/pkg/provider/aws/client_test.go | 162 +++++++++++++++++++ 1 file changed, 162 insertions(+) diff --git a/runtime_scan/pkg/provider/aws/client_test.go b/runtime_scan/pkg/provider/aws/client_test.go index 2df680a96..4f9cc1bfb 100644 --- a/runtime_scan/pkg/provider/aws/client_test.go +++ b/runtime_scan/pkg/provider/aws/client_test.go @@ -525,3 +525,165 @@ func Test_getInstancesFromDescribeInstancesOutput(t *testing.T) { }) } } + +func Test_createInstanceStateFilters(t *testing.T) { + type args struct { + scanStopped bool + } + tests := []struct { + name string + args args + want []ec2types.Filter + }{ + { + name: "should scan stopped", + args: args{ + scanStopped: true, + }, + want: []ec2types.Filter{ + { + Name: utils.StringPtr(instanceStateFilterName), + Values: []string{"running", "stopped"}, + }, + }, + }, + { + name: "should not scan stopped", + args: args{ + scanStopped: false, + }, + want: []ec2types.Filter{ + { + Name: utils.StringPtr(instanceStateFilterName), + Values: []string{"running"}, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := createInstanceStateFilters(tt.args.scanStopped); !reflect.DeepEqual(got, tt.want) { + t.Errorf("createInstanceStateFilters() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_getInstanceState(t *testing.T) { + type args struct { + result *ec2.DescribeInstancesOutput + instanceID string + } + tests := []struct { + name string + args args + want ec2types.InstanceStateName + }{ + { + name: "state running", + args: args{ + result: &ec2.DescribeInstancesOutput{ + Reservations: []ec2types.Reservation{ + { + Instances: []ec2types.Instance{ + { + InstanceId: utils.StringPtr("instance-1"), + }, + { + InstanceId: utils.StringPtr("instance-2"), + }, + }, + }, + { + Instances: []ec2types.Instance{ + { + InstanceId: utils.StringPtr("instance-3"), + State: &ec2types.InstanceState{ + Name: ec2types.InstanceStateNameRunning, + }, + }, + }, + }, + }, + }, + instanceID: "instance-3", + }, + want: ec2types.InstanceStateNameRunning, + }, + { + name: "state pending", + args: args{ + result: &ec2.DescribeInstancesOutput{ + Reservations: []ec2types.Reservation{ + { + Instances: []ec2types.Instance{ + { + InstanceId: utils.StringPtr("instance-1"), + }, + { + InstanceId: utils.StringPtr("instance-2"), + State: &ec2types.InstanceState{ + Name: ec2types.InstanceStateNamePending, + }, + }, + }, + }, + { + Instances: []ec2types.Instance{ + { + InstanceId: utils.StringPtr("instance-3"), + State: &ec2types.InstanceState{ + Name: ec2types.InstanceStateNameRunning, + }, + }, + }, + }, + }, + }, + instanceID: "instance-2", + }, + want: ec2types.InstanceStateNamePending, + }, + { + name: "instance id not found", + args: args{ + result: &ec2.DescribeInstancesOutput{ + Reservations: []ec2types.Reservation{ + { + Instances: []ec2types.Instance{ + { + InstanceId: utils.StringPtr("instance-1"), + }, + { + InstanceId: utils.StringPtr("instance-2"), + State: &ec2types.InstanceState{ + Name: ec2types.InstanceStateNamePending, + }, + }, + }, + }, + { + Instances: []ec2types.Instance{ + { + InstanceId: utils.StringPtr("instance-3"), + State: &ec2types.InstanceState{ + Name: ec2types.InstanceStateNameRunning, + }, + }, + }, + }, + }, + }, + instanceID: "instance-4", + }, + want: ec2types.InstanceStateNamePending, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := getInstanceState(tt.args.result, tt.args.instanceID); got != tt.want { + t.Errorf("getInstanceState() = %v, want %v", got, tt.want) + } + }) + } +} \ No newline at end of file From b3efb7f23203137010eda4d100c8c4f4bd1090b5 Mon Sep 17 00:00:00 2001 From: Erez Fishhimer Date: Sun, 13 Nov 2022 15:21:25 +0200 Subject: [PATCH 10/23] move scanner logic to the specific provider client --- runtime_scan/pkg/provider/aws/client.go | 53 ++++++++++++++++++ runtime_scan/pkg/provider/aws/client_test.go | 2 +- runtime_scan/pkg/provider/provider.go | 29 +++------- runtime_scan/pkg/scanner/job_managment.go | 56 ++++---------------- runtime_scan/pkg/types/types.go | 8 +++ 5 files changed, 78 insertions(+), 70 deletions(-) diff --git a/runtime_scan/pkg/provider/aws/client.go b/runtime_scan/pkg/provider/aws/client.go index bb0dfde0f..593a31944 100644 --- a/runtime_scan/pkg/provider/aws/client.go +++ b/runtime_scan/pkg/provider/aws/client.go @@ -98,6 +98,59 @@ func (c *Client) Discover(scope *types.ScanScope) ([]types.Instance, error) { return ret, nil } +func (c *Client) RunScanningJob(config types.JobConfig) (types.Job, error) { + rootVolume, err := c.GetInstanceRootVolume(config.InstanceToScan) + if err != nil { + return types.Job{}, fmt.Errorf("failed to get instance root volume. instance id=%v: %v", config.InstanceToScan.ID, err) + } + + // create a snapshot of the root volume + srcSnapshot, err := c.CreateSnapshot(rootVolume) + if err != nil { + return types.Job{}, fmt.Errorf("failed to create snapshot: %v", err) + } + if err := c.WaitForSnapshotReady(srcSnapshot); err != nil { + return types.Job{}, fmt.Errorf("failed to wait for snapshot to be ready: %v", err) + } + + //copy the snapshot to the scanner region + // TODO check if scanner region is same as snapshot region? + cpySnapshot, err := c.CopySnapshot(srcSnapshot, config.Region) + if err != nil { + return types.Job{}, fmt.Errorf("failed to copy snapshot: %v", err) + } + if err := c.WaitForSnapshotReady(cpySnapshot); err != nil { + return types.Job{}, fmt.Errorf("failed to wait for snapshot to be ready: %v", err) + } + + // create the scanner job (vm) with a boot script + launchedInstance, err := c.LaunchInstance(config.ImageID, config.DeviceName, config.SubnetID, cpySnapshot) + if err != nil { + return types.Job{}, fmt.Errorf("failed to launch instance: %v", err) + } + if err := c.WaitForInstanceReady(launchedInstance); err != nil { + return types.Job{}, fmt.Errorf("failed to wait for instance to be ready: %v", err) + } + + return types.Job{ + Instance: launchedInstance, + SrcSnapshot: srcSnapshot, + DstSnapshot: cpySnapshot, + }, nil +} + +func (c *Client) DeleteJob(job types.Job) { + if err := c.DeleteInstance(job.Instance); err != nil { + log.Errorf("failed to delete instance: %v", err) + } + if err := c.DeleteSnapshot(job.SrcSnapshot); err != nil { + log.Errorf("failed to delete source snapshot: %v", err) + } + if err := c.DeleteSnapshot(job.DstSnapshot); err != nil { + log.Errorf("failed to delete dest snapshot: %v", err) + } +} + func (c *Client) GetInstances(filters []ec2types.Filter, excludeTags []*types.Tag, regionID string) ([]types.Instance, error) { var ret []types.Instance diff --git a/runtime_scan/pkg/provider/aws/client_test.go b/runtime_scan/pkg/provider/aws/client_test.go index 4f9cc1bfb..2d57b6530 100644 --- a/runtime_scan/pkg/provider/aws/client_test.go +++ b/runtime_scan/pkg/provider/aws/client_test.go @@ -686,4 +686,4 @@ func Test_getInstanceState(t *testing.T) { } }) } -} \ No newline at end of file +} diff --git a/runtime_scan/pkg/provider/provider.go b/runtime_scan/pkg/provider/provider.go index 046d391e4..468d38fe2 100644 --- a/runtime_scan/pkg/provider/provider.go +++ b/runtime_scan/pkg/provider/provider.go @@ -15,28 +15,13 @@ package provider -import types2 "github.com/openclarity/vmclarity/runtime_scan/pkg/types" +import "github.com/openclarity/vmclarity/runtime_scan/pkg/types" type Client interface { - // Discover - list VM instances in the account according to the filters. - Discover(filters *types2.ScanScope) ([]types2.Instance, error) - // Create a snapshot of a volume. - // return the newly created snapshot. - CreateSnapshot(volume types2.Volume) (types2.Snapshot, error) - // CopySnapshot - copy the snapshot from src region to dest region. - // return the newly created (copy) snapshot. - CopySnapshot(snapshot types2.Snapshot, dstRegion string) (types2.Snapshot, error) - // WaitForSnapshotReady - wait until snapshot state is 'completed'. - WaitForSnapshotReady(snapshot types2.Snapshot) error - // WaitForSnapshotReady - wait until instance state is 'running'. - WaitForInstanceReady(instance types2.Instance) error - // GetInstanceRootVolume - get the instance's root volume. - GetInstanceRootVolume(instance types2.Instance) (types2.Volume, error) - // LaunchInstance - launch an instance. the snapshot will be attached to the instance at launch. - // return the launched instance - LaunchInstance(ami, deviceName, subnetID string, snapshot types2.Snapshot) (types2.Instance, error) - // DeleteInstance - delete an instance. - DeleteInstance(instance types2.Instance) error - // DeleteSnapshot - delete a snapshot. - DeleteSnapshot(snapshot types2.Snapshot) error + // Discover - list VM instances in the account according to the scan scope. + Discover(*types.ScanScope) ([]types.Instance, error) + // RunScanningJob - run a scanning job + RunScanningJob(config types.JobConfig) (types.Job, error) + // DeleteJob - delete a job + DeleteJob(types.Job) } diff --git a/runtime_scan/pkg/scanner/job_managment.go b/runtime_scan/pkg/scanner/job_managment.go index 09cba61bd..984ca0d95 100644 --- a/runtime_scan/pkg/scanner/job_managment.go +++ b/runtime_scan/pkg/scanner/job_managment.go @@ -147,44 +147,14 @@ func (s *Scanner) waitForResult(data *scanData, ks chan bool) { } func (s *Scanner) runJob(data *scanData) (types.Job, error) { - rootVolume, err := s.providerClient.GetInstanceRootVolume(data.instance) - if err != nil { - return types.Job{}, fmt.Errorf("failed to get instance root volume. instance id=%v: %v", data.instance.ID, err) - } - - // create a snapshot of the root volume - srcSnapshot, err := s.providerClient.CreateSnapshot(rootVolume) - if err != nil { - return types.Job{}, fmt.Errorf("failed to create snapshot: %v", err) - } - if err := s.providerClient.WaitForSnapshotReady(srcSnapshot); err != nil { - return types.Job{}, fmt.Errorf("failed to wait for snapshot to be ready: %v", err) - } - - //copy the snapshot to the scanner region - // TODO check if scanner region is same as snapshot region? - cpySnapshot, err := s.providerClient.CopySnapshot(srcSnapshot, s.region) - if err != nil { - return types.Job{}, fmt.Errorf("failed to copy snapshot: %v", err) - } - if err := s.providerClient.WaitForSnapshotReady(cpySnapshot); err != nil { - return types.Job{}, fmt.Errorf("failed to wait for snapshot to be ready: %v", err) - } - - // create the scanner job (vm) with a boot script - launchedInstance, err := s.providerClient.LaunchInstance(s.jobAMI, s.deviceName, s.subnetID, cpySnapshot) - if err != nil { - return types.Job{}, fmt.Errorf("failed to launch instance: %v", err) - } - if err := s.providerClient.WaitForInstanceReady(launchedInstance); err != nil { - return types.Job{}, fmt.Errorf("failed to wait for instance to be ready: %v", err) - } - - return types.Job{ - Instance: launchedInstance, - SrcSnapshot: srcSnapshot, - DstSnapshot: cpySnapshot, - }, nil + jobConfig := types.JobConfig{ + InstanceToScan: data.instance, + Region: s.region, + ImageID: s.jobAMI, + DeviceName: s.deviceName, + SubnetID: s.subnetID, + } + return s.providerClient.RunScanningJob(jobConfig) } func (s *Scanner) deleteJobIfNeeded(job *types.Job, isSuccessfulJob, isCompletedJob bool) { @@ -211,13 +181,5 @@ func (s *Scanner) deleteJobIfNeeded(job *types.Job, isSuccessfulJob, isCompleted } func (s *Scanner) deleteJob(job *types.Job) { - if err := s.providerClient.DeleteInstance(job.Instance); err != nil { - log.Errorf("failed to delete instance: %v", err) - } - if err := s.providerClient.DeleteSnapshot(job.SrcSnapshot); err != nil { - log.Errorf("failed to delete source snapshot: %v", err) - } - if err := s.providerClient.DeleteSnapshot(job.DstSnapshot); err != nil { - log.Errorf("failed to delete dest snapshot: %v", err) - } + s.providerClient.DeleteJob(*job) } diff --git a/runtime_scan/pkg/types/types.go b/runtime_scan/pkg/types/types.go index 19eb2fde3..9b370a7d7 100644 --- a/runtime_scan/pkg/types/types.go +++ b/runtime_scan/pkg/types/types.go @@ -84,6 +84,14 @@ type Job struct { DstSnapshot Snapshot } +type JobConfig struct { + InstanceToScan Instance + Region string + ImageID string + DeviceName string + SubnetID string +} + type Instance struct { ID string Region string From 048701f070d4f1325852f2f75aab0eb2aa2fe178 Mon Sep 17 00:00:00 2001 From: Erez Fishhimer Date: Sun, 13 Nov 2022 15:24:17 +0200 Subject: [PATCH 11/23] remove comment --- runtime_scan/pkg/provider/aws/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime_scan/pkg/provider/aws/client.go b/runtime_scan/pkg/provider/aws/client.go index 593a31944..ae6dd8d0c 100644 --- a/runtime_scan/pkg/provider/aws/client.go +++ b/runtime_scan/pkg/provider/aws/client.go @@ -430,7 +430,7 @@ func getVPCSecurityGroupsIDs(vpc types.VPC) []string { const ( vpcIDFilterName = "vpc-id" - sgIDFilterName = "instance.group-id" // TODO is this the right one? + sgIDFilterName = "instance.group-id" instanceStateFilterName = "instance-state-name" ) From 311b738495577d12359e7918acc5f6fae472f82d Mon Sep 17 00:00:00 2001 From: Erez Fishhimer Date: Sun, 13 Nov 2022 22:13:13 +0200 Subject: [PATCH 12/23] simplify --- runtime_scan/pkg/provider/aws/client.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/runtime_scan/pkg/provider/aws/client.go b/runtime_scan/pkg/provider/aws/client.go index ae6dd8d0c..28807c9c0 100644 --- a/runtime_scan/pkg/provider/aws/client.go +++ b/runtime_scan/pkg/provider/aws/client.go @@ -490,12 +490,7 @@ func (c *Client) getRegionsToScan(scope *types.ScanScope) ([]types.Region, error return c.ListAllRegions() } - var ret []types.Region - for _, region := range scope.Regions { - ret = append(ret, region) - } - - return ret, nil + return scope.Regions, nil } func (c *Client) ListAllRegions() ([]types.Region, error) { From 827ba2172f058645c1909e08de041454954d07f5 Mon Sep 17 00:00:00 2001 From: Erez Fishhimer Date: Thu, 17 Nov 2022 13:53:13 +0200 Subject: [PATCH 13/23] review --- runtime_scan/pkg/config/aws/aws_config.go | 51 +++ runtime_scan/pkg/config/config.go | 25 +- runtime_scan/pkg/config/scan_config.go | 7 +- runtime_scan/pkg/orchestrator/orchestrator.go | 16 +- runtime_scan/pkg/provider/aws/client.go | 341 ++++-------------- runtime_scan/pkg/provider/aws/client_test.go | 327 +++++++++-------- runtime_scan/pkg/provider/aws/instance.go | 115 ++++++ runtime_scan/pkg/provider/aws/snapshot.go | 105 ++++++ runtime_scan/pkg/provider/aws/volume.go | 57 +++ runtime_scan/pkg/provider/provider.go | 12 +- runtime_scan/pkg/provider/types.go | 70 ++++ runtime_scan/pkg/scanner/job_managment.go | 82 +++-- runtime_scan/pkg/scanner/scanner.go | 12 +- runtime_scan/pkg/types/types.go | 59 +-- 14 files changed, 719 insertions(+), 560 deletions(-) create mode 100644 runtime_scan/pkg/config/aws/aws_config.go create mode 100644 runtime_scan/pkg/provider/aws/instance.go create mode 100644 runtime_scan/pkg/provider/aws/snapshot.go create mode 100644 runtime_scan/pkg/provider/aws/volume.go create mode 100644 runtime_scan/pkg/provider/types.go diff --git a/runtime_scan/pkg/config/aws/aws_config.go b/runtime_scan/pkg/config/aws/aws_config.go new file mode 100644 index 000000000..f1b39c564 --- /dev/null +++ b/runtime_scan/pkg/config/aws/aws_config.go @@ -0,0 +1,51 @@ +// Copyright © 2022 Cisco Systems, Inc. and its affiliates. +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aws + +import "github.com/spf13/viper" + +const ( + AWSSubnetID = "AWS_SUBNET_ID" + AWSJobImageID = "AWS_JOB_IMAGE_ID" + defaultAWSJobImageID = "ami-0568773882d492fc8" // ubuntu server 22.04 LTS (HVM), SSD volume type + AWSAttachedVolumeDeviceName = "AWS_ATTACHED_VOLUME_DEVICE_NAME" + defaultAttachedVolumeDeviceName = "xvdh" +) + +type Config struct { + AmiID string // image id of a scanner job + DeviceName string // the name of the block device to attach to the scanner instance (mounted snapshot) + SubnetID string // the scanner's subnet ID +} + +func setConfigDefaults() { + viper.SetDefault(AWSJobImageID, defaultAWSJobImageID) + viper.SetDefault(AWSAttachedVolumeDeviceName, defaultAttachedVolumeDeviceName) + + viper.AutomaticEnv() +} + +func LoadConfig() *Config { + setConfigDefaults() + + config := &Config{ + AmiID: viper.GetString(AWSJobImageID), + DeviceName: viper.GetString(AWSAttachedVolumeDeviceName), + SubnetID: viper.GetString(AWSSubnetID), + } + + return config +} diff --git a/runtime_scan/pkg/config/config.go b/runtime_scan/pkg/config/config.go index 019070650..0afc73f27 100644 --- a/runtime_scan/pkg/config/config.go +++ b/runtime_scan/pkg/config/config.go @@ -17,32 +17,26 @@ package config import ( "github.com/spf13/viper" + + "github.com/openclarity/vmclarity/runtime_scan/pkg/config/aws" ) const ( + ScannerAWSRegion = "SCANNER_AWS_REGION" + defaultScannerAWSRegion = "us-east-1" ScannerJobResultListenPort = "SCANNER_JOB_RESULT_LISTEN_PORT" - ScannerRegion = "SCANNER_REGION" - defaultScannerRegion = "us-east-1" - ScannerJobImageID = "SCANNER_JOB_IMAGE_ID" - defaultScannerJobImageID = "ami-0568773882d492fc8" // ubuntu server 22.04 LTS (HVM), SSD volume type - ScannerAttachedVolumeDeviceName = "SCANNER_ATTACHED_VOLUME_DEVICE_NAME" - defaultAttachedVolumeDeviceName = "xvdh" defaultScannerJobResultListenPort = 8888 ) type Config struct { ScannerJobResultListenPort int - Region string // scanner region - AmiID string // image id of a scanner job - DeviceName string // the name of the block device to attach to the scanner instance (mounted snapshot) - SubnetID string // the scanner's subnet ID + Region string // scanner region + AWSConfig *aws.Config } func setConfigDefaults() { + viper.SetDefault(ScannerAWSRegion, defaultScannerAWSRegion) viper.SetDefault(ScannerJobResultListenPort, defaultScannerJobResultListenPort) - viper.SetDefault(ScannerRegion, defaultScannerRegion) - viper.SetDefault(ScannerJobImageID, defaultScannerJobImageID) - viper.SetDefault(ScannerAttachedVolumeDeviceName, defaultAttachedVolumeDeviceName) viper.AutomaticEnv() } @@ -52,9 +46,8 @@ func LoadConfig() (*Config, error) { config := &Config{ ScannerJobResultListenPort: viper.GetInt(ScannerJobResultListenPort), - Region: viper.GetString(ScannerRegion), - AmiID: viper.GetString(ScannerJobImageID), - DeviceName: viper.GetString(ScannerAttachedVolumeDeviceName), + Region: viper.GetString(ScannerAWSRegion), + AWSConfig: aws.LoadConfig(), } return config, nil diff --git a/runtime_scan/pkg/config/scan_config.go b/runtime_scan/pkg/config/scan_config.go index 65dfeb0da..ea5c8e18c 100644 --- a/runtime_scan/pkg/config/scan_config.go +++ b/runtime_scan/pkg/config/scan_config.go @@ -21,6 +21,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/spf13/viper" + "github.com/openclarity/vmclarity/runtime_scan/pkg/provider" "github.com/openclarity/vmclarity/runtime_scan/pkg/types" ) @@ -33,15 +34,15 @@ const ( type ScanConfig struct { MaxScanParallelism int // instances to scan - Instances []types.Instance + Instances []provider.Instance ScanScope types.ScanScope JobResultTimeout time.Duration DeleteJobPolicy DeleteJobPolicyType } func setScanConfigDefaults() { - viper.SetDefault(MaxParallelism, "10") - viper.SetDefault(JobResultTimeout, "10m") + viper.SetDefault(MaxParallelism, "5") + viper.SetDefault(JobResultTimeout, "120m") viper.SetDefault(DeleteJobPolicy, DeleteJobPolicySuccessful) viper.AutomaticEnv() diff --git a/runtime_scan/pkg/orchestrator/orchestrator.go b/runtime_scan/pkg/orchestrator/orchestrator.go index e31426556..0dc3a4bac 100644 --- a/runtime_scan/pkg/orchestrator/orchestrator.go +++ b/runtime_scan/pkg/orchestrator/orchestrator.go @@ -16,6 +16,7 @@ package orchestrator import ( + "context" "fmt" "sync" @@ -23,7 +24,6 @@ import ( _config "github.com/openclarity/vmclarity/runtime_scan/pkg/config" "github.com/openclarity/vmclarity/runtime_scan/pkg/provider" - "github.com/openclarity/vmclarity/runtime_scan/pkg/provider/aws" _scanner "github.com/openclarity/vmclarity/runtime_scan/pkg/scanner" "github.com/openclarity/vmclarity/runtime_scan/pkg/types" ) @@ -46,17 +46,11 @@ type VulnerabilitiesScanner interface { Stop() } -func Create(config *_config.Config) (*Orchestrator, error) { - // for now will statically create aws client here (until we support more cloud providers) - awsClient, err := aws.Create() - if err != nil { - return nil, fmt.Errorf("failed to create aws client: %v", err) - } - +func Create(config *_config.Config, client provider.Client) (*Orchestrator, error) { orc := &Orchestrator{ - scanner: _scanner.CreateScanner(config, awsClient), + scanner: _scanner.CreateScanner(config, client), config: config, - providerClient: awsClient, + providerClient: client, Mutex: sync.Mutex{}, } @@ -77,7 +71,7 @@ func (o *Orchestrator) Stop() { } func (o *Orchestrator) Scan(scanConfig *_config.ScanConfig, scanDone chan struct{}) error { - instances, err := o.providerClient.Discover(&scanConfig.ScanScope) + instances, err := o.providerClient.Discover(context.TODO(), &scanConfig.ScanScope) if err != nil { return err } diff --git a/runtime_scan/pkg/provider/aws/client.go b/runtime_scan/pkg/provider/aws/client.go index 28807c9c0..6227ea4cd 100644 --- a/runtime_scan/pkg/provider/aws/client.go +++ b/runtime_scan/pkg/provider/aws/client.go @@ -19,19 +19,21 @@ import ( "context" "fmt" "strings" - "time" awsconfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/ec2" ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" log "github.com/sirupsen/logrus" + "github.com/openclarity/vmclarity/runtime_scan/pkg/config/aws" + "github.com/openclarity/vmclarity/runtime_scan/pkg/provider" "github.com/openclarity/vmclarity/runtime_scan/pkg/types" "github.com/openclarity/vmclarity/runtime_scan/pkg/utils" ) type Client struct { ec2Client *ec2.Client + awsConfig *aws.Config } var ( @@ -46,10 +48,12 @@ var ( } ) -func Create() (*Client, error) { - var awsClient Client +func Create(ctx context.Context, config *aws.Config) (*Client, error) { + var awsClient = Client{ + awsConfig: config, + } - cfg, err := awsconfig.LoadDefaultConfig(context.TODO()) + cfg, err := awsconfig.LoadDefaultConfig(ctx) if err != nil { return nil, fmt.Errorf("failed to load aws config: %v", err) } @@ -59,11 +63,11 @@ func Create() (*Client, error) { return &awsClient, nil } -func (c *Client) Discover(scope *types.ScanScope) ([]types.Instance, error) { - var ret []types.Instance +func (c *Client) Discover(ctx context.Context, scope *types.ScanScope) ([]provider.Instance, error) { + var ret []provider.Instance var filters []ec2types.Filter - regions, err := c.getRegionsToScan(scope) + regions, err := c.getRegionsToScan(ctx, scope) if err != nil { return nil, fmt.Errorf("failed to get regions to scan: %v", err) } @@ -76,7 +80,7 @@ func (c *Client) Discover(scope *types.ScanScope) ([]types.Instance, error) { for _, region := range regions { // if no vpcs, that mean that we don't need any vpc filters if len(region.VPCs) == 0 { - instances, err := c.GetInstances(filters, scope.ExcludeTags, region.ID) + instances, err := c.GetInstances(ctx, filters, scope.ExcludeTags, region.ID) if err != nil { return nil, fmt.Errorf("failed to get instances: %v", err) } @@ -88,7 +92,7 @@ func (c *Client) Discover(scope *types.ScanScope) ([]types.Instance, error) { for _, vpc := range region.VPCs { vpcFilters := append(filters, createVPCFilters(vpc)...) - instances, err := c.GetInstances(vpcFilters, scope.ExcludeTags, region.ID) + instances, err := c.GetInstances(ctx, vpcFilters, scope.ExcludeTags, region.ID) if err != nil { return nil, fmt.Errorf("failed to get instances: %v", err) } @@ -98,244 +102,19 @@ func (c *Client) Discover(scope *types.ScanScope) ([]types.Instance, error) { return ret, nil } -func (c *Client) RunScanningJob(config types.JobConfig) (types.Job, error) { - rootVolume, err := c.GetInstanceRootVolume(config.InstanceToScan) - if err != nil { - return types.Job{}, fmt.Errorf("failed to get instance root volume. instance id=%v: %v", config.InstanceToScan.ID, err) - } - - // create a snapshot of the root volume - srcSnapshot, err := c.CreateSnapshot(rootVolume) - if err != nil { - return types.Job{}, fmt.Errorf("failed to create snapshot: %v", err) - } - if err := c.WaitForSnapshotReady(srcSnapshot); err != nil { - return types.Job{}, fmt.Errorf("failed to wait for snapshot to be ready: %v", err) - } - - //copy the snapshot to the scanner region - // TODO check if scanner region is same as snapshot region? - cpySnapshot, err := c.CopySnapshot(srcSnapshot, config.Region) - if err != nil { - return types.Job{}, fmt.Errorf("failed to copy snapshot: %v", err) - } - if err := c.WaitForSnapshotReady(cpySnapshot); err != nil { - return types.Job{}, fmt.Errorf("failed to wait for snapshot to be ready: %v", err) - } - - // create the scanner job (vm) with a boot script - launchedInstance, err := c.LaunchInstance(config.ImageID, config.DeviceName, config.SubnetID, cpySnapshot) - if err != nil { - return types.Job{}, fmt.Errorf("failed to launch instance: %v", err) - } - if err := c.WaitForInstanceReady(launchedInstance); err != nil { - return types.Job{}, fmt.Errorf("failed to wait for instance to be ready: %v", err) - } - - return types.Job{ - Instance: launchedInstance, - SrcSnapshot: srcSnapshot, - DstSnapshot: cpySnapshot, - }, nil -} - -func (c *Client) DeleteJob(job types.Job) { - if err := c.DeleteInstance(job.Instance); err != nil { - log.Errorf("failed to delete instance: %v", err) - } - if err := c.DeleteSnapshot(job.SrcSnapshot); err != nil { - log.Errorf("failed to delete source snapshot: %v", err) - } - if err := c.DeleteSnapshot(job.DstSnapshot); err != nil { - log.Errorf("failed to delete dest snapshot: %v", err) - } -} - -func (c *Client) GetInstances(filters []ec2types.Filter, excludeTags []*types.Tag, regionID string) ([]types.Instance, error) { - var ret []types.Instance - - out, err := c.ec2Client.DescribeInstances(context.TODO(), &ec2.DescribeInstancesInput{ - Filters: filters, - MaxResults: utils.Int32Ptr(50), // TODO what will be a good number? - }, func(options *ec2.Options) { - options.Region = regionID - }) - if err != nil { - return nil, fmt.Errorf("failed to describe instances: %v", err) - } - ret = append(ret, getInstancesFromDescribeInstancesOutput(out, excludeTags, regionID)...) - - // use pagination - for out.NextToken != nil { - out, err = c.ec2Client.DescribeInstances(context.TODO(), &ec2.DescribeInstancesInput{ - Filters: filters, - MaxResults: utils.Int32Ptr(50), - NextToken: out.NextToken, - }, func(options *ec2.Options) { - options.Region = regionID - }) - if err != nil { - return nil, fmt.Errorf("failed to describe instances: %v", err) - } - ret = append(ret, getInstancesFromDescribeInstancesOutput(out, excludeTags, regionID)...) - } - - return ret, nil -} - -func (c *Client) CreateSnapshot(volume types.Volume) (types.Snapshot, error) { - params := ec2.CreateSnapshotInput{ - VolumeId: &volume.ID, - Description: &snapshotDescription, - TagSpecifications: []ec2types.TagSpecification{ - { - ResourceType: ec2types.ResourceTypeSnapshot, - Tags: vmclarityTags, - }, - }, - } - out, err := c.ec2Client.CreateSnapshot(context.TODO(), ¶ms, func(options *ec2.Options) { - options.Region = volume.Region - }) - if err != nil { - return types.Snapshot{}, fmt.Errorf("failed to create snapshot: %v", err) - } - return types.Snapshot{ - ID: *out.SnapshotId, - Region: volume.Region, - }, nil -} - -func (c *Client) WaitForSnapshotReady(snapshot types.Snapshot) error { - ticker := time.NewTicker(3 * time.Second) - defer ticker.Stop() - timeout := time.After(3 * time.Minute) - - for { - select { - case <-ticker.C: - out, err := c.ec2Client.DescribeSnapshots(context.TODO(), &ec2.DescribeSnapshotsInput{ - SnapshotIds: []string{snapshot.ID}, - }, func(options *ec2.Options) { - options.Region = snapshot.Region - }) - if err != nil { - return fmt.Errorf("failed to describe snapshot. snapshotID=%v: %v", snapshot.ID, err) - } - if len(out.Snapshots) != 1 { - return fmt.Errorf("got unexcpected number of snapshots (%v) with snapshot id %v. excpecting 1", len(out.Snapshots), snapshot.ID) - } - if out.Snapshots[0].State == ec2types.SnapshotStateCompleted { - return nil - } - case <-timeout: - return fmt.Errorf("timeout") - } - } -} - -func (c *Client) WaitForInstanceReady(instance types.Instance) error { - ticker := time.NewTicker(3 * time.Second) - defer ticker.Stop() - timeout := time.After(3 * time.Minute) - - for { - select { - case <-ticker.C: - out, err := c.ec2Client.DescribeInstances(context.TODO(), &ec2.DescribeInstancesInput{ - InstanceIds: []string{instance.ID}, - }, func(options *ec2.Options) { - options.Region = instance.Region - }) - if err != nil { - return fmt.Errorf("failed to describe instance. instanceID=%v: %v", instance.ID, err) - } - state := getInstanceState(out, instance.ID) - if state == ec2types.InstanceStateNameRunning { - return nil - } - case <-timeout: - return fmt.Errorf("timeout") - } - } -} - -func (c *Client) CopySnapshot(snapshot types.Snapshot, dstRegion string) (types.Snapshot, error) { - snap, err := c.ec2Client.CopySnapshot(context.TODO(), &ec2.CopySnapshotInput{ - SourceRegion: &snapshot.Region, - SourceSnapshotId: &snapshot.ID, - Description: &snapshotDescription, - TagSpecifications: []ec2types.TagSpecification{ - { - ResourceType: ec2types.ResourceTypeSnapshot, - Tags: vmclarityTags, - }, - }, - }, func(options *ec2.Options) { - options.Region = dstRegion - }) - if err != nil { - return types.Snapshot{}, fmt.Errorf("failed to copy snapshot: %v", err) - } - - return types.Snapshot{ - ID: *snap.SnapshotId, - Region: dstRegion, - }, nil -} - -func (c *Client) GetInstanceRootVolume(instance types.Instance) (types.Volume, error) { - out, err := c.ec2Client.DescribeInstances(context.TODO(), &ec2.DescribeInstancesInput{ - InstanceIds: []string{instance.ID}, - }, func(options *ec2.Options) { - options.Region = instance.Region - }) - if err != nil { - return types.Volume{}, fmt.Errorf("failed to describe instances: %v", err) - } - - if len(out.Reservations) == 0 { - return types.Volume{}, fmt.Errorf("no reservations were found") - } - if len(out.Reservations) > 1 { - return types.Volume{}, fmt.Errorf("more than one reservations were found") - } - if len(out.Reservations[0].Instances) == 0 { - return types.Volume{}, fmt.Errorf("no instances were found") - } - if len(out.Reservations[0].Instances) > 1 { - return types.Volume{}, fmt.Errorf("more than one instances were found") - } - - outInstance := out.Reservations[0].Instances[0] - rootDeviceName := *outInstance.RootDeviceName - - // find root volume of the instance - for _, blkDevice := range outInstance.BlockDeviceMappings { - if strings.Compare(*blkDevice.DeviceName, rootDeviceName) == 0 { - return types.Volume{ - ID: *blkDevice.Ebs.VolumeId, - Name: rootDeviceName, - Region: instance.Region, - }, nil - } - } - return types.Volume{}, fmt.Errorf("failed to find root device volume") -} - -func (c *Client) LaunchInstance(ami, deviceName, subnetID string, snapshot types.Snapshot) (types.Instance, error) { - out, err := c.ec2Client.RunInstances(context.TODO(), &ec2.RunInstancesInput{ +func (c *Client) LaunchInstance(ctx context.Context, snapshot provider.Snapshot) (provider.Instance, error) { + out, err := c.ec2Client.RunInstances(ctx, &ec2.RunInstancesInput{ MaxCount: utils.Int32Ptr(1), MinCount: utils.Int32Ptr(1), - ImageId: &ami, + ImageId: &c.awsConfig.AmiID, BlockDeviceMappings: []ec2types.BlockDeviceMapping{ { // attach the snapshot to the instance at launch (a new volume will be created) - DeviceName: &deviceName, + DeviceName: &c.awsConfig.DeviceName, Ebs: &ec2types.EbsBlockDevice{ DeleteOnTermination: utils.BoolPtr(true), Encrypted: nil, // ? - SnapshotId: &snapshot.ID, + SnapshotId: utils.StringPtr(snapshot.GetID()), VolumeSize: nil, // default is snapshot size VolumeType: ec2types.VolumeTypeGp2, // TODO need to decide volume type }, @@ -343,7 +122,7 @@ func (c *Client) LaunchInstance(ami, deviceName, subnetID string, snapshot types }, InstanceType: ec2types.InstanceTypeT2Large, // TODO need to decide instance type SecurityGroups: nil, // use default for now - SubnetId: &subnetID, + SubnetId: &c.awsConfig.SubnetID, TagSpecifications: []ec2types.TagSpecification{ { ResourceType: ec2types.ResourceTypeInstance, @@ -352,42 +131,49 @@ func (c *Client) LaunchInstance(ami, deviceName, subnetID string, snapshot types }, UserData: nil, // TODO put launch script here }, func(options *ec2.Options) { - options.Region = snapshot.Region + options.Region = snapshot.GetRegion() }) if err != nil { - return types.Instance{}, fmt.Errorf("failed to run instances: %v", err) + return nil, fmt.Errorf("failed to run instances: %v", err) } - return types.Instance{ - ID: *out.Instances[0].InstanceId, - Region: snapshot.Region, + return &InstanceImpl{ + ec2Client: c.ec2Client, + id: *out.Instances[0].InstanceId, + region: snapshot.GetRegion(), }, nil } -func (c *Client) DeleteInstance(instance types.Instance) error { - _, err := c.ec2Client.TerminateInstances(context.TODO(), &ec2.TerminateInstancesInput{ - InstanceIds: []string{instance.ID}, +func (c *Client) GetInstances(ctx context.Context, filters []ec2types.Filter, excludeTags []*provider.Tag, regionID string) ([]provider.Instance, error) { + var ret []provider.Instance + + out, err := c.ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{ + Filters: filters, + MaxResults: utils.Int32Ptr(50), // TODO what will be a good number? }, func(options *ec2.Options) { - options.Region = instance.Region + options.Region = regionID }) if err != nil { - return fmt.Errorf("failed to terminate instances: %v", err) + return nil, fmt.Errorf("failed to describe instances: %v", err) } + ret = append(ret, c.getInstancesFromDescribeInstancesOutput(out, excludeTags, regionID)...) - return nil -} - -func (c *Client) DeleteSnapshot(snapshot types.Snapshot) error { - _, err := c.ec2Client.DeleteSnapshot(context.TODO(), &ec2.DeleteSnapshotInput{ - SnapshotId: &snapshot.ID, - }, func(options *ec2.Options) { - options.Region = snapshot.Region - }) - if err != nil { - return fmt.Errorf("failed to delete snapshot: %v", err) + // use pagination + for out.NextToken != nil { + out, err = c.ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{ + Filters: filters, + MaxResults: utils.Int32Ptr(50), + NextToken: out.NextToken, + }, func(options *ec2.Options) { + options.Region = regionID + }) + if err != nil { + return nil, fmt.Errorf("failed to describe instances: %v", err) + } + ret = append(ret, c.getInstancesFromDescribeInstancesOutput(out, excludeTags, regionID)...) } - return nil + return ret, nil } func getInstanceState(result *ec2.DescribeInstancesOutput, instanceID string) ec2types.InstanceStateName { @@ -403,24 +189,25 @@ func getInstanceState(result *ec2.DescribeInstancesOutput, instanceID string) ec return ec2types.InstanceStateNamePending } -func getInstancesFromDescribeInstancesOutput(result *ec2.DescribeInstancesOutput, excludeTags []*types.Tag, regionID string) []types.Instance { - var ret []types.Instance +func (c *Client) getInstancesFromDescribeInstancesOutput(result *ec2.DescribeInstancesOutput, excludeTags []*provider.Tag, regionID string) []provider.Instance { + var ret []provider.Instance for _, reservation := range result.Reservations { for _, instance := range reservation.Instances { if hasExcludeTags(excludeTags, instance.Tags) { continue } - ret = append(ret, types.Instance{ - ID: *instance.InstanceId, - Region: regionID, + ret = append(ret, &InstanceImpl{ + ec2Client: c.ec2Client, + id: *instance.InstanceId, + region: regionID, }) } } return ret } -func getVPCSecurityGroupsIDs(vpc types.VPC) []string { +func getVPCSecurityGroupsIDs(vpc provider.VPC) []string { var sgs []string for _, sg := range vpc.SecurityGroups { sgs = append(sgs, sg.ID) @@ -434,7 +221,7 @@ const ( instanceStateFilterName = "instance-state-name" ) -func createVPCFilters(vpc types.VPC) []ec2types.Filter { +func createVPCFilters(vpc provider.VPC) []ec2types.Filter { var ret = make([]ec2types.Filter, 0) var sgs []string @@ -472,7 +259,7 @@ func createInstanceStateFilters(scanStopped bool) []ec2types.Filter { return filters } -func createInclusionTagsFilters(tags []*types.Tag) []ec2types.Filter { +func createInclusionTagsFilters(tags []*provider.Tag) []ec2types.Filter { var filters []ec2types.Filter for _, tag := range tags { @@ -485,31 +272,31 @@ func createInclusionTagsFilters(tags []*types.Tag) []ec2types.Filter { return filters } -func (c *Client) getRegionsToScan(scope *types.ScanScope) ([]types.Region, error) { +func (c *Client) getRegionsToScan(ctx context.Context, scope *types.ScanScope) ([]provider.Region, error) { if scope.All { - return c.ListAllRegions() + return c.ListAllRegions(ctx) } return scope.Regions, nil } -func (c *Client) ListAllRegions() ([]types.Region, error) { - var ret []types.Region - out, err := c.ec2Client.DescribeRegions(context.TODO(), &ec2.DescribeRegionsInput{ +func (c *Client) ListAllRegions(ctx context.Context) ([]provider.Region, error) { + var ret []provider.Region + out, err := c.ec2Client.DescribeRegions(ctx, &ec2.DescribeRegionsInput{ AllRegions: nil, // display also disabled regions? }) if err != nil { return nil, fmt.Errorf("failed to describe regions: %v", err) } for _, region := range out.Regions { - ret = append(ret, types.Region{ + ret = append(ret, provider.Region{ ID: *region.RegionName, }) } return ret, nil } -func hasExcludeTags(excludeTags []*types.Tag, instanceTags []ec2types.Tag) bool { +func hasExcludeTags(excludeTags []*provider.Tag, instanceTags []ec2types.Tag) bool { var excludedTagsMap = make(map[string]string) for _, tag := range excludeTags { diff --git a/runtime_scan/pkg/provider/aws/client_test.go b/runtime_scan/pkg/provider/aws/client_test.go index 2d57b6530..d38af2d9f 100644 --- a/runtime_scan/pkg/provider/aws/client_test.go +++ b/runtime_scan/pkg/provider/aws/client_test.go @@ -17,13 +17,12 @@ package aws import ( "reflect" - "sort" "testing" "github.com/aws/aws-sdk-go-v2/service/ec2" ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/openclarity/vmclarity/runtime_scan/pkg/types" + "github.com/openclarity/vmclarity/runtime_scan/pkg/provider" "github.com/openclarity/vmclarity/runtime_scan/pkg/utils" ) @@ -133,7 +132,7 @@ func Test_createVPCFilters(t *testing.T) { ) type args struct { - vpc types.VPC + vpc provider.VPC } tests := []struct { name string @@ -143,7 +142,7 @@ func Test_createVPCFilters(t *testing.T) { { name: "vpc with no security group", args: args{ - vpc: types.VPC{ + vpc: provider.VPC{ ID: vpcID, SecurityGroups: nil, }, @@ -158,9 +157,9 @@ func Test_createVPCFilters(t *testing.T) { { name: "vpc with one security group", args: args{ - vpc: types.VPC{ + vpc: provider.VPC{ ID: vpcID, - SecurityGroups: []types.SecurityGroup{ + SecurityGroups: []provider.SecurityGroup{ { ID: sgID1, }, @@ -181,9 +180,9 @@ func Test_createVPCFilters(t *testing.T) { { name: "vpc with two security groups", args: args{ - vpc: types.VPC{ + vpc: provider.VPC{ ID: vpcID, - SecurityGroups: []types.SecurityGroup{ + SecurityGroups: []provider.SecurityGroup{ { ID: sgID1, }, @@ -222,7 +221,7 @@ func Test_createInclusionTagsFilters(t *testing.T) { ) type args struct { - tags []*types.Tag + tags []*provider.Tag } tests := []struct { name string @@ -239,7 +238,7 @@ func Test_createInclusionTagsFilters(t *testing.T) { { name: "1 tag", args: args{ - tags: []*types.Tag{ + tags: []*provider.Tag{ { Key: tagName, Val: tagVal, @@ -272,7 +271,7 @@ func Test_hasExcludedTags(t *testing.T) { ) type args struct { - excludeTags []*types.Tag + excludeTags []*provider.Tag instanceTags []ec2types.Tag } tests := []struct { @@ -283,7 +282,7 @@ func Test_hasExcludedTags(t *testing.T) { { name: "instance has no tags", args: args{ - excludeTags: []*types.Tag{ + excludeTags: []*provider.Tag{ { Key: tagName1, Val: tagVal1, @@ -317,7 +316,7 @@ func Test_hasExcludedTags(t *testing.T) { { name: "instance has excluded tags", args: args{ - excludeTags: []*types.Tag{ + excludeTags: []*provider.Tag{ { Key: tagName1, Val: tagVal1, @@ -343,7 +342,7 @@ func Test_hasExcludedTags(t *testing.T) { { name: "instance does not have excluded tags", args: args{ - excludeTags: []*types.Tag{ + excludeTags: []*provider.Tag{ { Key: "stam1", Val: "stam2", @@ -375,156 +374,156 @@ func Test_hasExcludedTags(t *testing.T) { }) } } - -func Test_getInstancesFromDescribeInstancesOutput(t *testing.T) { - type args struct { - result *ec2.DescribeInstancesOutput - excludeTags []*types.Tag - regionID string - } - tests := []struct { - name string - args args - want []types.Instance - }{ - { - name: "no reservations found", - args: args{ - result: &ec2.DescribeInstancesOutput{ - Reservations: []ec2types.Reservation{}, - }, - excludeTags: nil, - regionID: "region-1", - }, - want: nil, - }, - { - name: "no excluded tags", - args: args{ - result: &ec2.DescribeInstancesOutput{ - Reservations: []ec2types.Reservation{ - { - Instances: []ec2types.Instance{ - { - InstanceId: utils.StringPtr("instance-1"), - Tags: []ec2types.Tag{ - { - Key: utils.StringPtr("key-1"), - Value: utils.StringPtr("val-1"), - }, - }, - }, - { - InstanceId: utils.StringPtr("instance-2"), - Tags: []ec2types.Tag{ - { - Key: utils.StringPtr("key-2"), - Value: utils.StringPtr("val-2"), - }, - }, - }, - }, - }, - { - Instances: []ec2types.Instance{ - { - InstanceId: utils.StringPtr("instance-3"), - }, - }, - }, - }, - }, - excludeTags: nil, - regionID: "region-1", - }, - want: []types.Instance{ - { - ID: "instance-1", - Region: "region-1", - }, - { - ID: "instance-2", - Region: "region-1", - }, - { - ID: "instance-3", - Region: "region-1", - }, - }, - }, - { - name: "one excluded instance", - args: args{ - result: &ec2.DescribeInstancesOutput{ - Reservations: []ec2types.Reservation{ - { - Instances: []ec2types.Instance{ - { - InstanceId: utils.StringPtr("instance-1"), - Tags: []ec2types.Tag{ - { - Key: utils.StringPtr("key-1"), - Value: utils.StringPtr("val-1"), - }, - }, - }, - { - InstanceId: utils.StringPtr("instance-2"), - Tags: []ec2types.Tag{ - { - Key: utils.StringPtr("key-2"), - Value: utils.StringPtr("val-2"), - }, - }, - }, - }, - }, - { - Instances: []ec2types.Instance{ - { - InstanceId: utils.StringPtr("instance-3"), - }, - }, - }, - }, - }, - excludeTags: []*types.Tag{ - { - Key: "key-1", - Val: "val-1", - }, - }, - regionID: "region-1", - }, - want: []types.Instance{ - { - ID: "instance-2", - Region: "region-1", - }, - { - ID: "instance-3", - Region: "region-1", - }, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := getInstancesFromDescribeInstancesOutput(tt.args.result, tt.args.excludeTags, tt.args.regionID) - - sort.Slice(got, func(i, j int) bool { - return got[i].ID > got[j].ID - }) - sort.Slice(tt.want, func(i, j int) bool { - return tt.want[i].ID > tt.want[j].ID - }) - - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("getInstancesFromDescribeInstancesOutput() = %v, want %v", got, tt.want) - } - }) - } -} +// +//func Test_getInstancesFromDescribeInstancesOutput(t *testing.T) { +// type args struct { +// result *ec2.DescribeInstancesOutput +// excludeTags []*provider.Tag +// regionID string +// } +// tests := []struct { +// name string +// args args +// want []provider.Instance +// }{ +// { +// name: "no reservations found", +// args: args{ +// result: &ec2.DescribeInstancesOutput{ +// Reservations: []ec2types.Reservation{}, +// }, +// excludeTags: nil, +// regionID: "region-1", +// }, +// want: nil, +// }, +// { +// name: "no excluded tags", +// args: args{ +// result: &ec2.DescribeInstancesOutput{ +// Reservations: []ec2types.Reservation{ +// { +// Instances: []ec2types.Instance{ +// { +// InstanceId: utils.StringPtr("instance-1"), +// Tags: []ec2types.Tag{ +// { +// Key: utils.StringPtr("key-1"), +// Value: utils.StringPtr("val-1"), +// }, +// }, +// }, +// { +// InstanceId: utils.StringPtr("instance-2"), +// Tags: []ec2types.Tag{ +// { +// Key: utils.StringPtr("key-2"), +// Value: utils.StringPtr("val-2"), +// }, +// }, +// }, +// }, +// }, +// { +// Instances: []ec2types.Instance{ +// { +// InstanceId: utils.StringPtr("instance-3"), +// }, +// }, +// }, +// }, +// }, +// excludeTags: nil, +// regionID: "region-1", +// }, +// want: []provider.Instance{ +// { +// ID: "instance-1", +// Region: "region-1", +// }, +// { +// ID: "instance-2", +// Region: "region-1", +// }, +// { +// ID: "instance-3", +// Region: "region-1", +// }, +// }, +// }, +// { +// name: "one excluded instance", +// args: args{ +// result: &ec2.DescribeInstancesOutput{ +// Reservations: []ec2types.Reservation{ +// { +// Instances: []ec2types.Instance{ +// { +// InstanceId: utils.StringPtr("instance-1"), +// Tags: []ec2types.Tag{ +// { +// Key: utils.StringPtr("key-1"), +// Value: utils.StringPtr("val-1"), +// }, +// }, +// }, +// { +// InstanceId: utils.StringPtr("instance-2"), +// Tags: []ec2types.Tag{ +// { +// Key: utils.StringPtr("key-2"), +// Value: utils.StringPtr("val-2"), +// }, +// }, +// }, +// }, +// }, +// { +// Instances: []ec2types.Instance{ +// { +// InstanceId: utils.StringPtr("instance-3"), +// }, +// }, +// }, +// }, +// }, +// excludeTags: []*provider.Tag{ +// { +// Key: "key-1", +// Val: "val-1", +// }, +// }, +// regionID: "region-1", +// }, +// want: []provider.Instance{ +// { +// : "instance-2", +// Region: "region-1", +// }, +// { +// ID: "instance-3", +// Region: "region-1", +// }, +// }, +// }, +// } +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// got := getInstancesFromDescribeInstancesOutput(tt.args.result, tt.args.excludeTags, tt.args.regionID) +// +// sort.Slice(got, func(i, j int) bool { +// return got[i].ID > got[j].ID +// }) +// sort.Slice(tt.want, func(i, j int) bool { +// return tt.want[i].ID > tt.want[j].ID +// }) +// +// if !reflect.DeepEqual(got, tt.want) { +// t.Errorf("getInstancesFromDescribeInstancesOutput() = %v, want %v", got, tt.want) +// } +// }) +// } +//} func Test_createInstanceStateFilters(t *testing.T) { type args struct { diff --git a/runtime_scan/pkg/provider/aws/instance.go b/runtime_scan/pkg/provider/aws/instance.go new file mode 100644 index 000000000..62dfcaec0 --- /dev/null +++ b/runtime_scan/pkg/provider/aws/instance.go @@ -0,0 +1,115 @@ +// Copyright © 2022 Cisco Systems, Inc. and its affiliates. +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aws + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + + "github.com/openclarity/vmclarity/runtime_scan/pkg/provider" +) + +type InstanceImpl struct { + ec2Client *ec2.Client + id string + region string +} + +func (i *InstanceImpl) GetID() string { + return i.id +} + +func (i *InstanceImpl) GetRootVolume(ctx context.Context) (provider.Volume, error) { + out, err := i.ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{ + InstanceIds: []string{i.id}, + }, func(options *ec2.Options) { + options.Region = i.region + }) + if err != nil { + return nil, fmt.Errorf("failed to describe instances: %v", err) + } + + if len(out.Reservations) == 0 { + return nil, fmt.Errorf("no reservations were found") + } + if len(out.Reservations) > 1 { + return nil, fmt.Errorf("more than one reservations were found") + } + if len(out.Reservations[0].Instances) == 0 { + return nil, fmt.Errorf("no instances were found") + } + if len(out.Reservations[0].Instances) > 1 { + return nil, fmt.Errorf("more than one instances were found") + } + + outInstance := out.Reservations[0].Instances[0] + rootDeviceName := *outInstance.RootDeviceName + + // find root volume of the instance + for _, blkDevice := range outInstance.BlockDeviceMappings { + if strings.Compare(*blkDevice.DeviceName, rootDeviceName) == 0 { + return &VolumeImpl{ + ec2Client: i.ec2Client, + id: *blkDevice.Ebs.VolumeId, + name: rootDeviceName, + region: i.region, + }, nil + } + } + return nil, fmt.Errorf("failed to find root device volume") +} + +func (i *InstanceImpl) WaitForReady(ctx context.Context) error { + ctxWithTimeout, _ := context.WithTimeout(context.Background(), 3*time.Minute) + + for { + select { + case <-time.After(3 * time.Second): + out, err := i.ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{ + InstanceIds: []string{i.id}, + }, func(options *ec2.Options) { + options.Region = i.region + }) + if err != nil { + return fmt.Errorf("failed to describe instance. instanceID=%v: %v", i.id, err) + } + state := getInstanceState(out, i.id) + if state == ec2types.InstanceStateNameRunning { + return nil + } + case <-ctxWithTimeout.Done(): + return ctxWithTimeout.Err() + } + } +} + +func (i *InstanceImpl) Delete(ctx context.Context) error { + _, err := i.ec2Client.TerminateInstances(ctx, &ec2.TerminateInstancesInput{ + InstanceIds: []string{i.id}, + }, func(options *ec2.Options) { + options.Region = i.region + }) + if err != nil { + return fmt.Errorf("failed to terminate instances: %v", err) + } + + return nil +} diff --git a/runtime_scan/pkg/provider/aws/snapshot.go b/runtime_scan/pkg/provider/aws/snapshot.go new file mode 100644 index 000000000..9578222a2 --- /dev/null +++ b/runtime_scan/pkg/provider/aws/snapshot.go @@ -0,0 +1,105 @@ +// Copyright © 2022 Cisco Systems, Inc. and its affiliates. +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aws + +import ( + "context" + "fmt" + "time" + + "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + + "github.com/openclarity/vmclarity/runtime_scan/pkg/provider" +) + +type SnapshotImpl struct { + ec2Client *ec2.Client + id string + region string +} + +func (s *SnapshotImpl) GetID() string { + return s.id +} + +func (s *SnapshotImpl) GetRegion() string { + return s.region +} + +func (s *SnapshotImpl) Copy(ctx context.Context, dstRegion string) (provider.Snapshot, error) { + snap, err := s.ec2Client.CopySnapshot(ctx, &ec2.CopySnapshotInput{ + SourceRegion: &s.region, + SourceSnapshotId: &s.id, + Description: &snapshotDescription, + TagSpecifications: []ec2types.TagSpecification{ + { + ResourceType: ec2types.ResourceTypeSnapshot, + Tags: vmclarityTags, + }, + }, + }, func(options *ec2.Options) { + options.Region = dstRegion + }) + if err != nil { + return nil, fmt.Errorf("failed to copy snapshot: %v", err) + } + + return &SnapshotImpl{ + ec2Client: s.ec2Client, + id: *snap.SnapshotId, + region: dstRegion, + }, nil +} + +func (s *SnapshotImpl) Delete(ctx context.Context) error { + _, err := s.ec2Client.DeleteSnapshot(ctx, &ec2.DeleteSnapshotInput{ + SnapshotId: &s.id, + }, func(options *ec2.Options) { + options.Region = s.region + }) + if err != nil { + return fmt.Errorf("failed to delete snapshot: %v", err) + } + + return nil +} + +func (s *SnapshotImpl) WaitForReady(ctx context.Context) error { + ctxWithTimeout, _ := context.WithTimeout(context.Background(), 3*time.Minute) + + for { + select { + case <-time.After(3 * time.Second): + out, err := s.ec2Client.DescribeSnapshots(ctx, &ec2.DescribeSnapshotsInput{ + SnapshotIds: []string{s.id}, + }, func(options *ec2.Options) { + options.Region = s.region + }) + if err != nil { + return fmt.Errorf("failed to describe snapshot. snapshotID=%v: %v", s.id, err) + } + if len(out.Snapshots) != 1 { + return fmt.Errorf("got unexcpected number of snapshots (%v) with snapshot id %v. excpecting 1", len(out.Snapshots), s.id) + } + if out.Snapshots[0].State == ec2types.SnapshotStateCompleted { + return nil + } + case <-ctxWithTimeout.Done(): + return ctxWithTimeout.Err() + } + } +} diff --git a/runtime_scan/pkg/provider/aws/volume.go b/runtime_scan/pkg/provider/aws/volume.go new file mode 100644 index 000000000..d2c15f679 --- /dev/null +++ b/runtime_scan/pkg/provider/aws/volume.go @@ -0,0 +1,57 @@ +// Copyright © 2022 Cisco Systems, Inc. and its affiliates. +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aws + +import ( + "context" + "fmt" + + "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + + "github.com/openclarity/vmclarity/runtime_scan/pkg/provider" +) + +type VolumeImpl struct { + ec2Client *ec2.Client + id string + name string + region string +} + +func (v *VolumeImpl) TakeSnapshot(ctx context.Context) (provider.Snapshot, error) { + params := ec2.CreateSnapshotInput{ + VolumeId: &v.id, + Description: &snapshotDescription, + TagSpecifications: []ec2types.TagSpecification{ + { + ResourceType: ec2types.ResourceTypeSnapshot, + Tags: vmclarityTags, + }, + }, + } + out, err := v.ec2Client.CreateSnapshot(ctx, ¶ms, func(options *ec2.Options) { + options.Region = v.region + }) + if err != nil { + return nil, fmt.Errorf("failed to create snapshot: %v", err) + } + return &SnapshotImpl{ + ec2Client: v.ec2Client, + id: *out.SnapshotId, + region: v.region, + }, nil +} diff --git a/runtime_scan/pkg/provider/provider.go b/runtime_scan/pkg/provider/provider.go index 468d38fe2..0968fe5ed 100644 --- a/runtime_scan/pkg/provider/provider.go +++ b/runtime_scan/pkg/provider/provider.go @@ -15,13 +15,15 @@ package provider -import "github.com/openclarity/vmclarity/runtime_scan/pkg/types" +import ( + "context" + + "github.com/openclarity/vmclarity/runtime_scan/pkg/types" +) type Client interface { // Discover - list VM instances in the account according to the scan scope. - Discover(*types.ScanScope) ([]types.Instance, error) + Discover(context.Context, *types.ScanScope) ([]Instance, error) // RunScanningJob - run a scanning job - RunScanningJob(config types.JobConfig) (types.Job, error) - // DeleteJob - delete a job - DeleteJob(types.Job) + LaunchInstance(ctx context.Context, snapshot Snapshot) (Instance, error) } diff --git a/runtime_scan/pkg/provider/types.go b/runtime_scan/pkg/provider/types.go new file mode 100644 index 000000000..a0f8c0e34 --- /dev/null +++ b/runtime_scan/pkg/provider/types.go @@ -0,0 +1,70 @@ +// Copyright © 2022 Cisco Systems, Inc. and its affiliates. +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package provider + +import "context" + +type Tag struct { + Key string + Val string +} + +type SecurityGroup struct { + ID string +} + +type Job struct { + Instance Instance + SrcSnapshot Snapshot + DstSnapshot Snapshot +} + +type JobConfig struct { + InstanceToScan Instance + Region string + ImageID string + DeviceName string + SubnetID string +} + +type VPC struct { + ID string + SecurityGroups []SecurityGroup +} + +type Region struct { + ID string + VPCs []VPC +} + +type Instance interface { + GetID() string + GetRootVolume(ctx context.Context) (Volume, error) + WaitForReady(ctx context.Context) error + Delete(ctx context.Context) error +} + +type Volume interface { + TakeSnapshot(ctx context.Context) (Snapshot, error) +} + +type Snapshot interface { + GetID() string + GetRegion() string + Copy(ctx context.Context, dstRegion string) (Snapshot, error) + Delete(ctx context.Context) error + WaitForReady(ctx context.Context) error +} diff --git a/runtime_scan/pkg/scanner/job_managment.go b/runtime_scan/pkg/scanner/job_managment.go index 984ca0d95..a77581827 100644 --- a/runtime_scan/pkg/scanner/job_managment.go +++ b/runtime_scan/pkg/scanner/job_managment.go @@ -16,6 +16,7 @@ package scanner import ( + "context" "fmt" "sync/atomic" "time" @@ -23,6 +24,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/openclarity/vmclarity/runtime_scan/pkg/config" + "github.com/openclarity/vmclarity/runtime_scan/pkg/provider" "github.com/openclarity/vmclarity/runtime_scan/pkg/types" ) @@ -69,7 +71,7 @@ func (s *Scanner) jobBatchManagement(scanDone chan struct{}) { case q <- data: atomic.AddUint32(instancesStartedToScan, 1) case <-ks: - log.WithFields(s.logFields).Debugf("Scan process was canceled. instanceID=%v, scanUUID=%v", data.instance.ID, data.scanUUID) + log.WithFields(s.logFields).Debugf("Scan process was canceled. instanceID=%v, scanUUID=%v", data.instance.GetID(), data.scanUUID) return } }(data, s.killSignal) @@ -91,7 +93,7 @@ func (s *Scanner) worker(queue chan *scanData, workNumber int, done, ks chan boo for { select { case data := <-queue: - job, err := s.runJob(data) + job, err := s.runJob(context.TODO(), data) if err != nil { errMsg := fmt.Errorf("failed to run job: %v", err) log.WithFields(s.logFields).Error(errMsg) @@ -108,12 +110,12 @@ func (s *Scanner) worker(queue chan *scanData, workNumber int, done, ks chan boo s.waitForResult(data, ks) } - s.deleteJobIfNeeded(&job, data.success, data.completed) + s.deleteJobIfNeeded(context.TODO(), &job, data.success, data.completed) select { case done <- true: case <-ks: - log.WithFields(s.logFields).Infof("Instance scan was canceled. instanceID=%v", data.instance.ID) + log.WithFields(s.logFields).Infof("Instance scan was canceled. instanceID=%v", data.instance.GetID()) } case <-ks: log.WithFields(s.logFields).Debugf("worker #%v halted", workNumber) @@ -123,13 +125,13 @@ func (s *Scanner) worker(queue chan *scanData, workNumber int, done, ks chan boo } func (s *Scanner) waitForResult(data *scanData, ks chan bool) { - log.WithFields(s.logFields).Infof("Waiting for result. instanceID=%+v", data.instance.ID) + log.WithFields(s.logFields).Infof("Waiting for result. instanceID=%+v", data.instance.GetID()) ticker := time.NewTicker(s.scanConfig.JobResultTimeout) select { case <-data.resultChan: - log.WithFields(s.logFields).Infof("Instance scanned result has arrived. instanceID=%v", data.instance.ID) + log.WithFields(s.logFields).Infof("Instance scanned result has arrived. instanceID=%v", data.instance.GetID()) case <-ticker.C: - errMsg := fmt.Errorf("job has timed out. instanceID=%v", data.instance.ID) + errMsg := fmt.Errorf("job has timed out. instanceID=%v", data.instance.GetID()) log.WithFields(s.logFields).Warn(errMsg) s.Lock() data.success = false @@ -142,29 +144,57 @@ func (s *Scanner) waitForResult(data *scanData, ks chan bool) { data.completed = true s.Unlock() case <-ks: - log.WithFields(s.logFields).Infof("Instance scan was canceled. instanceID=%v", data.instance.ID) + log.WithFields(s.logFields).Infof("Instance scan was canceled. instanceID=%v", data.instance.GetID()) } } -func (s *Scanner) runJob(data *scanData) (types.Job, error) { - jobConfig := types.JobConfig{ - InstanceToScan: data.instance, - Region: s.region, - ImageID: s.jobAMI, - DeviceName: s.deviceName, - SubnetID: s.subnetID, +func (s *Scanner) runJob(ctx context.Context, data *scanData) (provider.Job, error) { + instanceToScan := data.instance + + volume, err := instanceToScan.GetRootVolume(ctx) + if err != nil { + return provider.Job{}, fmt.Errorf("failed to get root volume of an instance %v: %v", instanceToScan.GetID(), err) + } + + snapshot, err := volume.TakeSnapshot(ctx) + if err != nil { + return provider.Job{}, fmt.Errorf("failed to take snapshot of a volume: %v", err) + + } + if err := snapshot.WaitForReady(ctx); err != nil { + return provider.Job{}, fmt.Errorf("failed to wait for snapshot %v ready: %v", snapshot.GetID(), err) + + } + + cpySnapshot, err := snapshot.Copy(ctx, s.region) + if err != nil { + return provider.Job{}, fmt.Errorf("failed to copy snapshot %v: %v", snapshot.GetID(), err) } - return s.providerClient.RunScanningJob(jobConfig) + + if err := cpySnapshot.WaitForReady(ctx); err != nil { + return provider.Job{}, fmt.Errorf("failed wait for snapshot %v ready: %v", cpySnapshot.GetID(), err) + } + + i, err := s.providerClient.LaunchInstance(ctx, cpySnapshot) + if err != nil { + return provider.Job{}, fmt.Errorf("failed to launch a new instance: %v", err) + } + + return provider.Job{ + Instance: i, + SrcSnapshot: snapshot, + DstSnapshot: cpySnapshot, + }, nil } -func (s *Scanner) deleteJobIfNeeded(job *types.Job, isSuccessfulJob, isCompletedJob bool) { +func (s *Scanner) deleteJobIfNeeded(ctx context.Context, job *provider.Job, isSuccessfulJob, isCompletedJob bool) { if job == nil { return } // delete uncompleted jobs - scan process was canceled if !isCompletedJob { - s.deleteJob(job) + s.deleteJob(ctx, job) return } @@ -172,14 +202,22 @@ func (s *Scanner) deleteJobIfNeeded(job *types.Job, isSuccessfulJob, isCompleted case config.DeleteJobPolicyNever: // do nothing case config.DeleteJobPolicyAll: - s.deleteJob(job) + s.deleteJob(ctx, job) case config.DeleteJobPolicySuccessful: if isSuccessfulJob { - s.deleteJob(job) + s.deleteJob(ctx, job) } } } -func (s *Scanner) deleteJob(job *types.Job) { - s.providerClient.DeleteJob(*job) +func (s *Scanner) deleteJob(ctx context.Context, job *provider.Job) { + if err := job.Instance.Delete(ctx); err != nil { + log.Errorf("failed to delete instance: %v", err) + } + if err := job.SrcSnapshot.Delete(ctx); err != nil { + log.Errorf("failed to delete source snapshot: %v", err) + } + if err := job.DstSnapshot.Delete(ctx); err != nil { + log.Errorf("failed to delete dest snapshot: %v", err) + } } diff --git a/runtime_scan/pkg/scanner/scanner.go b/runtime_scan/pkg/scanner/scanner.go index e738a10d0..bb517fff2 100644 --- a/runtime_scan/pkg/scanner/scanner.go +++ b/runtime_scan/pkg/scanner/scanner.go @@ -37,16 +37,13 @@ type Scanner struct { providerClient provider.Client logFields log.Fields - region string - jobAMI string - deviceName string - subnetID string + region string sync.Mutex } type scanData struct { - instance types.Instance + instance provider.Instance scanUUID string vulnerabilitiesResult vulnerabilitiesScanResult resultChan chan bool @@ -72,9 +69,6 @@ func CreateScanner(config *_config.Config, providerClient provider.Client) *Scan providerClient: providerClient, logFields: log.Fields{"scanner id": uuid.NewV4().String()}, region: config.Region, - jobAMI: config.AmiID, - deviceName: config.DeviceName, - subnetID: config.SubnetID, Mutex: sync.Mutex{}, } @@ -88,7 +82,7 @@ func (s *Scanner) initScan() error { // Populate the instance to scanData map for _, instance := range s.scanConfig.Instances { - instanceIDToScanData[instance.ID] = &scanData{ + instanceIDToScanData[instance.GetID()] = &scanData{ instance: instance, scanUUID: uuid.NewV4().String(), vulnerabilitiesResult: vulnerabilitiesScanResult{}, diff --git a/runtime_scan/pkg/types/types.go b/runtime_scan/pkg/types/types.go index 9b370a7d7..a74863049 100644 --- a/runtime_scan/pkg/types/types.go +++ b/runtime_scan/pkg/types/types.go @@ -15,12 +15,14 @@ package types +import "github.com/openclarity/vmclarity/runtime_scan/pkg/provider" + type ScanScope struct { All bool - Regions []Region + Regions []provider.Region ScanStopped bool - IncludeTags []*Tag - ExcludeTags []*Tag + IncludeTags []*provider.Tag + ExcludeTags []*provider.Tag } type Status string @@ -47,7 +49,7 @@ func (s *ScanProgress) SetStatus(status Status) { type InstanceScanResult struct { // Instance data - Instance Instance + Instance provider.Instance // Scan results Vulnerabilities []string // TODO define vulnerabilities struct Success bool @@ -58,52 +60,3 @@ type ScanResults struct { InstanceScanResults []*InstanceScanResult Progress ScanProgress } - -type Tag struct { - Key string - Val string -} - -type SecurityGroup struct { - ID string -} - -type VPC struct { - ID string - SecurityGroups []SecurityGroup -} - -type Region struct { - ID string - VPCs []VPC -} - -type Job struct { - Instance Instance - SrcSnapshot Snapshot - DstSnapshot Snapshot -} - -type JobConfig struct { - InstanceToScan Instance - Region string - ImageID string - DeviceName string - SubnetID string -} - -type Instance struct { - ID string - Region string -} - -type Snapshot struct { - ID string - Region string -} - -type Volume struct { - ID string - Name string - Region string -} From 84fc1deea6a7e5332a5dbc5c6b646bc0c75cb0fb Mon Sep 17 00:00:00 2001 From: Erez Fishhimer Date: Thu, 17 Nov 2022 14:13:33 +0200 Subject: [PATCH 14/23] wip --- runtime_scan/pkg/config/aws/{aws_config.go => config.go} | 0 runtime_scan/pkg/config/config.go | 8 ++++---- 2 files changed, 4 insertions(+), 4 deletions(-) rename runtime_scan/pkg/config/aws/{aws_config.go => config.go} (100%) diff --git a/runtime_scan/pkg/config/aws/aws_config.go b/runtime_scan/pkg/config/aws/config.go similarity index 100% rename from runtime_scan/pkg/config/aws/aws_config.go rename to runtime_scan/pkg/config/aws/config.go diff --git a/runtime_scan/pkg/config/config.go b/runtime_scan/pkg/config/config.go index 0afc73f27..7463db875 100644 --- a/runtime_scan/pkg/config/config.go +++ b/runtime_scan/pkg/config/config.go @@ -22,15 +22,15 @@ import ( ) const ( - ScannerAWSRegion = "SCANNER_AWS_REGION" - defaultScannerAWSRegion = "us-east-1" + ScannerAWSRegion = "SCANNER_AWS_REGION" + defaultScannerAWSRegion = "us-east-1" ScannerJobResultListenPort = "SCANNER_JOB_RESULT_LISTEN_PORT" defaultScannerJobResultListenPort = 8888 ) type Config struct { ScannerJobResultListenPort int - Region string // scanner region + Region string // scanner region AWSConfig *aws.Config } @@ -46,7 +46,7 @@ func LoadConfig() (*Config, error) { config := &Config{ ScannerJobResultListenPort: viper.GetInt(ScannerJobResultListenPort), - Region: viper.GetString(ScannerAWSRegion), + Region: viper.GetString(ScannerAWSRegion), AWSConfig: aws.LoadConfig(), } From a5c4676746af139aac06e10abc6afb5afba32e19 Mon Sep 17 00:00:00 2001 From: Erez Fishhimer Date: Sun, 20 Nov 2022 09:09:14 +0200 Subject: [PATCH 15/23] move aws specific types to aws package --- runtime_scan/pkg/config/scan_config.go | 5 +- runtime_scan/pkg/provider/aws/client.go | 60 +-- runtime_scan/pkg/provider/aws/client_test.go | 395 +++++++++--------- runtime_scan/pkg/provider/aws/instance.go | 8 +- runtime_scan/pkg/provider/aws/snapshot.go | 8 +- runtime_scan/pkg/provider/aws/types.go | 28 ++ runtime_scan/pkg/provider/aws/volume.go | 4 +- .../pkg/provider/{provider.go => client.go} | 4 +- runtime_scan/pkg/provider/types.go | 70 ---- runtime_scan/pkg/scanner/job_managment.go | 39 +- runtime_scan/pkg/scanner/scanner.go | 2 +- runtime_scan/pkg/types/types.go | 46 +- 12 files changed, 343 insertions(+), 326 deletions(-) create mode 100644 runtime_scan/pkg/provider/aws/types.go rename runtime_scan/pkg/provider/{provider.go => client.go} (85%) delete mode 100644 runtime_scan/pkg/provider/types.go diff --git a/runtime_scan/pkg/config/scan_config.go b/runtime_scan/pkg/config/scan_config.go index ea5c8e18c..7e62e595d 100644 --- a/runtime_scan/pkg/config/scan_config.go +++ b/runtime_scan/pkg/config/scan_config.go @@ -21,7 +21,6 @@ import ( log "github.com/sirupsen/logrus" "github.com/spf13/viper" - "github.com/openclarity/vmclarity/runtime_scan/pkg/provider" "github.com/openclarity/vmclarity/runtime_scan/pkg/types" ) @@ -34,8 +33,8 @@ const ( type ScanConfig struct { MaxScanParallelism int // instances to scan - Instances []provider.Instance - ScanScope types.ScanScope + Instances []types.Instance + ScanScope interface{} JobResultTimeout time.Duration DeleteJobPolicy DeleteJobPolicyType } diff --git a/runtime_scan/pkg/provider/aws/client.go b/runtime_scan/pkg/provider/aws/client.go index 6227ea4cd..efbaa255d 100644 --- a/runtime_scan/pkg/provider/aws/client.go +++ b/runtime_scan/pkg/provider/aws/client.go @@ -26,7 +26,6 @@ import ( log "github.com/sirupsen/logrus" "github.com/openclarity/vmclarity/runtime_scan/pkg/config/aws" - "github.com/openclarity/vmclarity/runtime_scan/pkg/provider" "github.com/openclarity/vmclarity/runtime_scan/pkg/types" "github.com/openclarity/vmclarity/runtime_scan/pkg/utils" ) @@ -63,9 +62,15 @@ func Create(ctx context.Context, config *aws.Config) (*Client, error) { return &awsClient, nil } -func (c *Client) Discover(ctx context.Context, scope *types.ScanScope) ([]provider.Instance, error) { - var ret []provider.Instance +func (c *Client) Discover(ctx context.Context, scanScope interface{}) ([]types.Instance, error) { + var ret []types.Instance var filters []ec2types.Filter + var scope *ScanScope + var ok bool + + if scope, ok = scanScope.(*ScanScope); !ok { + return nil, fmt.Errorf("failed to assert scope type") + } regions, err := c.getRegionsToScan(ctx, scope) if err != nil { @@ -79,8 +84,8 @@ func (c *Client) Discover(ctx context.Context, scope *types.ScanScope) ([]provid for _, region := range regions { // if no vpcs, that mean that we don't need any vpc filters - if len(region.VPCs) == 0 { - instances, err := c.GetInstances(ctx, filters, scope.ExcludeTags, region.ID) + if len(region.vpcs) == 0 { + instances, err := c.GetInstances(ctx, filters, scope.ExcludeTags, region.Id) if err != nil { return nil, fmt.Errorf("failed to get instances: %v", err) } @@ -89,10 +94,10 @@ func (c *Client) Discover(ctx context.Context, scope *types.ScanScope) ([]provid } // need to do a per vpc call for DescribeInstances - for _, vpc := range region.VPCs { + for _, vpc := range region.vpcs { vpcFilters := append(filters, createVPCFilters(vpc)...) - instances, err := c.GetInstances(ctx, vpcFilters, scope.ExcludeTags, region.ID) + instances, err := c.GetInstances(ctx, vpcFilters, scope.ExcludeTags, region.Id) if err != nil { return nil, fmt.Errorf("failed to get instances: %v", err) } @@ -102,7 +107,7 @@ func (c *Client) Discover(ctx context.Context, scope *types.ScanScope) ([]provid return ret, nil } -func (c *Client) LaunchInstance(ctx context.Context, snapshot provider.Snapshot) (provider.Instance, error) { +func (c *Client) LaunchInstance(ctx context.Context, snapshot types.Snapshot) (types.Instance, error) { out, err := c.ec2Client.RunInstances(ctx, &ec2.RunInstancesInput{ MaxCount: utils.Int32Ptr(1), MinCount: utils.Int32Ptr(1), @@ -144,8 +149,8 @@ func (c *Client) LaunchInstance(ctx context.Context, snapshot provider.Snapshot) }, nil } -func (c *Client) GetInstances(ctx context.Context, filters []ec2types.Filter, excludeTags []*provider.Tag, regionID string) ([]provider.Instance, error) { - var ret []provider.Instance +func (c *Client) GetInstances(ctx context.Context, filters []ec2types.Filter, excludeTags []Tag, regionID string) ([]types.Instance, error) { + var ret []types.Instance out, err := c.ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{ Filters: filters, @@ -159,6 +164,7 @@ func (c *Client) GetInstances(ctx context.Context, filters []ec2types.Filter, ex ret = append(ret, c.getInstancesFromDescribeInstancesOutput(out, excludeTags, regionID)...) // use pagination + // TODO we can make it better by not saving all results in memory. See https://github.com/openclarity/vmclarity/pull/3#discussion_r1021656861 for out.NextToken != nil { out, err = c.ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{ Filters: filters, @@ -189,8 +195,8 @@ func getInstanceState(result *ec2.DescribeInstancesOutput, instanceID string) ec return ec2types.InstanceStateNamePending } -func (c *Client) getInstancesFromDescribeInstancesOutput(result *ec2.DescribeInstancesOutput, excludeTags []*provider.Tag, regionID string) []provider.Instance { - var ret []provider.Instance +func (c *Client) getInstancesFromDescribeInstancesOutput(result *ec2.DescribeInstancesOutput, excludeTags []Tag, regionID string) []types.Instance { + var ret []types.Instance for _, reservation := range result.Reservations { for _, instance := range reservation.Instances { @@ -207,10 +213,10 @@ func (c *Client) getInstancesFromDescribeInstancesOutput(result *ec2.DescribeIns return ret } -func getVPCSecurityGroupsIDs(vpc provider.VPC) []string { +func getVPCSecurityGroupsIDs(vpc VPC) []string { var sgs []string - for _, sg := range vpc.SecurityGroups { - sgs = append(sgs, sg.ID) + for _, sg := range vpc.securityGroups { + sgs = append(sgs, sg.id) } return sgs } @@ -221,14 +227,14 @@ const ( instanceStateFilterName = "instance-state-name" ) -func createVPCFilters(vpc provider.VPC) []ec2types.Filter { +func createVPCFilters(vpc VPC) []ec2types.Filter { var ret = make([]ec2types.Filter, 0) var sgs []string // create per vpc filters ret = append(ret, ec2types.Filter{ Name: utils.StringPtr(vpcIDFilterName), - Values: []string{vpc.ID}, + Values: []string{vpc.Id}, }) sgs = getVPCSecurityGroupsIDs(vpc) if len(sgs) > 0 { @@ -259,20 +265,20 @@ func createInstanceStateFilters(scanStopped bool) []ec2types.Filter { return filters } -func createInclusionTagsFilters(tags []*provider.Tag) []ec2types.Filter { +func createInclusionTagsFilters(tags []Tag) []ec2types.Filter { var filters []ec2types.Filter for _, tag := range tags { filters = append(filters, ec2types.Filter{ - Name: utils.StringPtr("tag:" + tag.Key), - Values: []string{tag.Val}, + Name: utils.StringPtr("tag:" + tag.key), + Values: []string{tag.val}, }) } return filters } -func (c *Client) getRegionsToScan(ctx context.Context, scope *types.ScanScope) ([]provider.Region, error) { +func (c *Client) getRegionsToScan(ctx context.Context, scope *ScanScope) ([]Region, error) { if scope.All { return c.ListAllRegions(ctx) } @@ -280,8 +286,8 @@ func (c *Client) getRegionsToScan(ctx context.Context, scope *types.ScanScope) ( return scope.Regions, nil } -func (c *Client) ListAllRegions(ctx context.Context) ([]provider.Region, error) { - var ret []provider.Region +func (c *Client) ListAllRegions(ctx context.Context) ([]Region, error) { + var ret []Region out, err := c.ec2Client.DescribeRegions(ctx, &ec2.DescribeRegionsInput{ AllRegions: nil, // display also disabled regions? }) @@ -289,18 +295,18 @@ func (c *Client) ListAllRegions(ctx context.Context) ([]provider.Region, error) return nil, fmt.Errorf("failed to describe regions: %v", err) } for _, region := range out.Regions { - ret = append(ret, provider.Region{ - ID: *region.RegionName, + ret = append(ret, Region{ + Id: *region.RegionName, }) } return ret, nil } -func hasExcludeTags(excludeTags []*provider.Tag, instanceTags []ec2types.Tag) bool { +func hasExcludeTags(excludeTags []Tag, instanceTags []ec2types.Tag) bool { var excludedTagsMap = make(map[string]string) for _, tag := range excludeTags { - excludedTagsMap[tag.Key] = tag.Val + excludedTagsMap[tag.key] = tag.val } for _, instanceTag := range instanceTags { if val, ok := excludedTagsMap[*instanceTag.Key]; ok { diff --git a/runtime_scan/pkg/provider/aws/client_test.go b/runtime_scan/pkg/provider/aws/client_test.go index d38af2d9f..bf468ddea 100644 --- a/runtime_scan/pkg/provider/aws/client_test.go +++ b/runtime_scan/pkg/provider/aws/client_test.go @@ -17,12 +17,12 @@ package aws import ( "reflect" + "sort" "testing" "github.com/aws/aws-sdk-go-v2/service/ec2" ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/openclarity/vmclarity/runtime_scan/pkg/provider" "github.com/openclarity/vmclarity/runtime_scan/pkg/utils" ) @@ -47,10 +47,10 @@ import ( // Regions: []types.Region{ // { // ID: "us-east-2", -// VPCs: []types.VPC{ +// vpcs: []types.VPC{ // { // ID: "vpc-32ea7c59", -// SecurityGroups: []types.SecurityGroup{ +// securityGroups: []types.SecurityGroup{ // { // ID: "sg-4d6b853a", // }, @@ -60,14 +60,14 @@ import ( // }, // { // ID: "us-east-1", -// VPCs: []types.VPC{ +// vpcs: []types.VPC{ // { // ID: "vpc-0c41450ba658eed00", -// SecurityGroups: nil, +// securityGroups: nil, // }, // { // ID: "vpc-9ca32ce1", -// SecurityGroups: []types.SecurityGroup{ +// securityGroups: []types.SecurityGroup{ // { // ID: "sg-030918e8e73254d42", // }, @@ -79,8 +79,8 @@ import ( // ScanStopped: true, // IncludeTags: []*types.Tag{ // { -// Key: "Name", -// Val: "diff-vpc", +// key: "Name", +// val: "diff-vpc", // }, // }, // ExcludeTags: nil, @@ -132,7 +132,7 @@ func Test_createVPCFilters(t *testing.T) { ) type args struct { - vpc provider.VPC + vpc VPC } tests := []struct { name string @@ -142,9 +142,9 @@ func Test_createVPCFilters(t *testing.T) { { name: "vpc with no security group", args: args{ - vpc: provider.VPC{ - ID: vpcID, - SecurityGroups: nil, + vpc: VPC{ + Id: vpcID, + securityGroups: nil, }, }, want: []ec2types.Filter{ @@ -157,11 +157,11 @@ func Test_createVPCFilters(t *testing.T) { { name: "vpc with one security group", args: args{ - vpc: provider.VPC{ - ID: vpcID, - SecurityGroups: []provider.SecurityGroup{ + vpc: VPC{ + Id: vpcID, + securityGroups: []SecurityGroup{ { - ID: sgID1, + id: sgID1, }, }, }, @@ -180,14 +180,14 @@ func Test_createVPCFilters(t *testing.T) { { name: "vpc with two security groups", args: args{ - vpc: provider.VPC{ - ID: vpcID, - SecurityGroups: []provider.SecurityGroup{ + vpc: VPC{ + Id: vpcID, + securityGroups: []SecurityGroup{ { - ID: sgID1, + id: sgID1, }, { - ID: sgID2, + id: sgID2, }, }, }, @@ -221,7 +221,7 @@ func Test_createInclusionTagsFilters(t *testing.T) { ) type args struct { - tags []*provider.Tag + tags []Tag } tests := []struct { name string @@ -238,10 +238,10 @@ func Test_createInclusionTagsFilters(t *testing.T) { { name: "1 tag", args: args{ - tags: []*provider.Tag{ + tags: []Tag{ { - Key: tagName, - Val: tagVal, + key: tagName, + val: tagVal, }, }, }, @@ -271,7 +271,7 @@ func Test_hasExcludedTags(t *testing.T) { ) type args struct { - excludeTags []*provider.Tag + excludeTags []Tag instanceTags []ec2types.Tag } tests := []struct { @@ -282,14 +282,14 @@ func Test_hasExcludedTags(t *testing.T) { { name: "instance has no tags", args: args{ - excludeTags: []*provider.Tag{ + excludeTags: []Tag{ { - Key: tagName1, - Val: tagVal1, + key: tagName1, + val: tagVal1, }, { - Key: "stam1", - Val: "stam2", + key: "stam1", + val: "stam2", }, }, instanceTags: nil, @@ -316,14 +316,14 @@ func Test_hasExcludedTags(t *testing.T) { { name: "instance has excluded tags", args: args{ - excludeTags: []*provider.Tag{ + excludeTags: []Tag{ { - Key: tagName1, - Val: tagVal1, + key: tagName1, + val: tagVal1, }, { - Key: "stam1", - Val: "stam2", + key: "stam1", + val: "stam2", }, }, instanceTags: []ec2types.Tag{ @@ -342,14 +342,14 @@ func Test_hasExcludedTags(t *testing.T) { { name: "instance does not have excluded tags", args: args{ - excludeTags: []*provider.Tag{ + excludeTags: []Tag{ { - Key: "stam1", - Val: "stam2", + key: "stam1", + val: "stam2", }, { - Key: "stam3", - Val: "stam4", + key: "stam3", + val: "stam4", }, }, instanceTags: []ec2types.Tag{ @@ -374,156 +374,6 @@ func Test_hasExcludedTags(t *testing.T) { }) } } -// -//func Test_getInstancesFromDescribeInstancesOutput(t *testing.T) { -// type args struct { -// result *ec2.DescribeInstancesOutput -// excludeTags []*provider.Tag -// regionID string -// } -// tests := []struct { -// name string -// args args -// want []provider.Instance -// }{ -// { -// name: "no reservations found", -// args: args{ -// result: &ec2.DescribeInstancesOutput{ -// Reservations: []ec2types.Reservation{}, -// }, -// excludeTags: nil, -// regionID: "region-1", -// }, -// want: nil, -// }, -// { -// name: "no excluded tags", -// args: args{ -// result: &ec2.DescribeInstancesOutput{ -// Reservations: []ec2types.Reservation{ -// { -// Instances: []ec2types.Instance{ -// { -// InstanceId: utils.StringPtr("instance-1"), -// Tags: []ec2types.Tag{ -// { -// Key: utils.StringPtr("key-1"), -// Value: utils.StringPtr("val-1"), -// }, -// }, -// }, -// { -// InstanceId: utils.StringPtr("instance-2"), -// Tags: []ec2types.Tag{ -// { -// Key: utils.StringPtr("key-2"), -// Value: utils.StringPtr("val-2"), -// }, -// }, -// }, -// }, -// }, -// { -// Instances: []ec2types.Instance{ -// { -// InstanceId: utils.StringPtr("instance-3"), -// }, -// }, -// }, -// }, -// }, -// excludeTags: nil, -// regionID: "region-1", -// }, -// want: []provider.Instance{ -// { -// ID: "instance-1", -// Region: "region-1", -// }, -// { -// ID: "instance-2", -// Region: "region-1", -// }, -// { -// ID: "instance-3", -// Region: "region-1", -// }, -// }, -// }, -// { -// name: "one excluded instance", -// args: args{ -// result: &ec2.DescribeInstancesOutput{ -// Reservations: []ec2types.Reservation{ -// { -// Instances: []ec2types.Instance{ -// { -// InstanceId: utils.StringPtr("instance-1"), -// Tags: []ec2types.Tag{ -// { -// Key: utils.StringPtr("key-1"), -// Value: utils.StringPtr("val-1"), -// }, -// }, -// }, -// { -// InstanceId: utils.StringPtr("instance-2"), -// Tags: []ec2types.Tag{ -// { -// Key: utils.StringPtr("key-2"), -// Value: utils.StringPtr("val-2"), -// }, -// }, -// }, -// }, -// }, -// { -// Instances: []ec2types.Instance{ -// { -// InstanceId: utils.StringPtr("instance-3"), -// }, -// }, -// }, -// }, -// }, -// excludeTags: []*provider.Tag{ -// { -// Key: "key-1", -// Val: "val-1", -// }, -// }, -// regionID: "region-1", -// }, -// want: []provider.Instance{ -// { -// : "instance-2", -// Region: "region-1", -// }, -// { -// ID: "instance-3", -// Region: "region-1", -// }, -// }, -// }, -// } -// for _, tt := range tests { -// t.Run(tt.name, func(t *testing.T) { -// got := getInstancesFromDescribeInstancesOutput(tt.args.result, tt.args.excludeTags, tt.args.regionID) -// -// sort.Slice(got, func(i, j int) bool { -// return got[i].ID > got[j].ID -// }) -// sort.Slice(tt.want, func(i, j int) bool { -// return tt.want[i].ID > tt.want[j].ID -// }) -// -// if !reflect.DeepEqual(got, tt.want) { -// t.Errorf("getInstancesFromDescribeInstancesOutput() = %v, want %v", got, tt.want) -// } -// }) -// } -//} func Test_createInstanceStateFilters(t *testing.T) { type args struct { @@ -686,3 +536,166 @@ func Test_getInstanceState(t *testing.T) { }) } } + +func TestClient_getInstancesFromDescribeInstancesOutput(t *testing.T) { + type fields struct {} + type args struct { + result *ec2.DescribeInstancesOutput + excludeTags []Tag + regionID string + } + tests := []struct { + name string + fields fields + args args + want []*InstanceImpl + }{ + { + name: "no reservations found", + args: args{ + result: &ec2.DescribeInstancesOutput{ + Reservations: []ec2types.Reservation{}, + }, + excludeTags: nil, + regionID: "region-1", + }, + want: nil, + }, + { + name: "no excluded tags", + args: args{ + result: &ec2.DescribeInstancesOutput{ + Reservations: []ec2types.Reservation{ + { + Instances: []ec2types.Instance{ + { + InstanceId: utils.StringPtr("instance-1"), + Tags: []ec2types.Tag{ + { + Key: utils.StringPtr("key-1"), + Value: utils.StringPtr("val-1"), + }, + }, + }, + { + InstanceId: utils.StringPtr("instance-2"), + Tags: []ec2types.Tag{ + { + Key: utils.StringPtr("key-2"), + Value: utils.StringPtr("val-2"), + }, + }, + }, + }, + }, + { + Instances: []ec2types.Instance{ + { + InstanceId: utils.StringPtr("instance-3"), + }, + }, + }, + }, + }, + excludeTags: nil, + regionID: "region-1", + }, + want: []*InstanceImpl{ + { + id: "instance-1", + region: "region-1", + }, + { + id: "instance-2", + region: "region-1", + }, + { + id: "instance-3", + region: "region-1", + }, + }, + }, + { + name: "one excluded instance", + args: args{ + result: &ec2.DescribeInstancesOutput{ + Reservations: []ec2types.Reservation{ + { + Instances: []ec2types.Instance{ + { + InstanceId: utils.StringPtr("instance-1"), + Tags: []ec2types.Tag{ + { + Key: utils.StringPtr("key-1"), + Value: utils.StringPtr("val-1"), + }, + }, + }, + { + InstanceId: utils.StringPtr("instance-2"), + Tags: []ec2types.Tag{ + { + Key: utils.StringPtr("key-2"), + Value: utils.StringPtr("val-2"), + }, + }, + }, + }, + }, + { + Instances: []ec2types.Instance{ + { + InstanceId: utils.StringPtr("instance-3"), + }, + }, + }, + }, + }, + excludeTags: []Tag{ + { + key: "key-1", + val: "val-1", + }, + }, + regionID: "region-1", + }, + want: []*InstanceImpl{ + { + id: "instance-2", + region: "region-1", + }, + { + id: "instance-3", + region: "region-1", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &Client{} + got := c.getInstancesFromDescribeInstancesOutput(tt.args.result, tt.args.excludeTags, tt.args.regionID) + + var gotInstances []*InstanceImpl + for _, instance := range got { + var instanceImpl *InstanceImpl + var ok bool + if instanceImpl, ok = instance.(*InstanceImpl); !ok { + t.Errorf("failed to convert type") + } + instanceImpl.ec2Client = nil + gotInstances = append(gotInstances, instanceImpl) + } + sort.Slice(gotInstances, func(i, j int) bool { + return gotInstances[i].id > gotInstances[j].id + }) + sort.Slice(tt.want, func(i, j int) bool { + return tt.want[i].id > tt.want[j].id + }) + + if !reflect.DeepEqual(gotInstances, tt.want) { + t.Errorf("getInstancesFromDescribeInstancesOutput() = %v, want %v", gotInstances, tt.want) + } + }) + } +} \ No newline at end of file diff --git a/runtime_scan/pkg/provider/aws/instance.go b/runtime_scan/pkg/provider/aws/instance.go index 62dfcaec0..d088dca87 100644 --- a/runtime_scan/pkg/provider/aws/instance.go +++ b/runtime_scan/pkg/provider/aws/instance.go @@ -24,7 +24,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/ec2" ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/openclarity/vmclarity/runtime_scan/pkg/provider" + "github.com/openclarity/vmclarity/runtime_scan/pkg/types" ) type InstanceImpl struct { @@ -37,7 +37,7 @@ func (i *InstanceImpl) GetID() string { return i.id } -func (i *InstanceImpl) GetRootVolume(ctx context.Context) (provider.Volume, error) { +func (i *InstanceImpl) GetRootVolume(ctx context.Context) (types.Volume, error) { out, err := i.ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{ InstanceIds: []string{i.id}, }, func(options *ec2.Options) { @@ -102,6 +102,10 @@ func (i *InstanceImpl) WaitForReady(ctx context.Context) error { } func (i *InstanceImpl) Delete(ctx context.Context) error { + if i == nil { + return nil + } + _, err := i.ec2Client.TerminateInstances(ctx, &ec2.TerminateInstancesInput{ InstanceIds: []string{i.id}, }, func(options *ec2.Options) { diff --git a/runtime_scan/pkg/provider/aws/snapshot.go b/runtime_scan/pkg/provider/aws/snapshot.go index 9578222a2..56e78a5df 100644 --- a/runtime_scan/pkg/provider/aws/snapshot.go +++ b/runtime_scan/pkg/provider/aws/snapshot.go @@ -23,7 +23,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/ec2" ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/openclarity/vmclarity/runtime_scan/pkg/provider" + "github.com/openclarity/vmclarity/runtime_scan/pkg/types" ) type SnapshotImpl struct { @@ -40,7 +40,7 @@ func (s *SnapshotImpl) GetRegion() string { return s.region } -func (s *SnapshotImpl) Copy(ctx context.Context, dstRegion string) (provider.Snapshot, error) { +func (s *SnapshotImpl) Copy(ctx context.Context, dstRegion string) (types.Snapshot, error) { snap, err := s.ec2Client.CopySnapshot(ctx, &ec2.CopySnapshotInput{ SourceRegion: &s.region, SourceSnapshotId: &s.id, @@ -66,6 +66,10 @@ func (s *SnapshotImpl) Copy(ctx context.Context, dstRegion string) (provider.Sna } func (s *SnapshotImpl) Delete(ctx context.Context) error { + if s == nil { + return nil + } + _, err := s.ec2Client.DeleteSnapshot(ctx, &ec2.DeleteSnapshotInput{ SnapshotId: &s.id, }, func(options *ec2.Options) { diff --git a/runtime_scan/pkg/provider/aws/types.go b/runtime_scan/pkg/provider/aws/types.go new file mode 100644 index 000000000..61d0ad064 --- /dev/null +++ b/runtime_scan/pkg/provider/aws/types.go @@ -0,0 +1,28 @@ +package aws + +type ScanScope struct { + All bool + Regions []Region + ScanStopped bool + IncludeTags []Tag + ExcludeTags []Tag +} + +type Tag struct { + key string + val string +} + +type SecurityGroup struct { + id string +} + +type VPC struct { + Id string + securityGroups []SecurityGroup +} + +type Region struct { + Id string + vpcs []VPC +} diff --git a/runtime_scan/pkg/provider/aws/volume.go b/runtime_scan/pkg/provider/aws/volume.go index d2c15f679..ec32a9800 100644 --- a/runtime_scan/pkg/provider/aws/volume.go +++ b/runtime_scan/pkg/provider/aws/volume.go @@ -22,7 +22,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/ec2" ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "github.com/openclarity/vmclarity/runtime_scan/pkg/provider" + "github.com/openclarity/vmclarity/runtime_scan/pkg/types" ) type VolumeImpl struct { @@ -32,7 +32,7 @@ type VolumeImpl struct { region string } -func (v *VolumeImpl) TakeSnapshot(ctx context.Context) (provider.Snapshot, error) { +func (v *VolumeImpl) TakeSnapshot(ctx context.Context) (types.Snapshot, error) { params := ec2.CreateSnapshotInput{ VolumeId: &v.id, Description: &snapshotDescription, diff --git a/runtime_scan/pkg/provider/provider.go b/runtime_scan/pkg/provider/client.go similarity index 85% rename from runtime_scan/pkg/provider/provider.go rename to runtime_scan/pkg/provider/client.go index 0968fe5ed..a01f4392c 100644 --- a/runtime_scan/pkg/provider/provider.go +++ b/runtime_scan/pkg/provider/client.go @@ -23,7 +23,7 @@ import ( type Client interface { // Discover - list VM instances in the account according to the scan scope. - Discover(context.Context, *types.ScanScope) ([]Instance, error) + Discover(context.Context, interface{}) ([]types.Instance, error) // RunScanningJob - run a scanning job - LaunchInstance(ctx context.Context, snapshot Snapshot) (Instance, error) + LaunchInstance(ctx context.Context, snapshot types.Snapshot) (types.Instance, error) } diff --git a/runtime_scan/pkg/provider/types.go b/runtime_scan/pkg/provider/types.go deleted file mode 100644 index a0f8c0e34..000000000 --- a/runtime_scan/pkg/provider/types.go +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright © 2022 Cisco Systems, Inc. and its affiliates. -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package provider - -import "context" - -type Tag struct { - Key string - Val string -} - -type SecurityGroup struct { - ID string -} - -type Job struct { - Instance Instance - SrcSnapshot Snapshot - DstSnapshot Snapshot -} - -type JobConfig struct { - InstanceToScan Instance - Region string - ImageID string - DeviceName string - SubnetID string -} - -type VPC struct { - ID string - SecurityGroups []SecurityGroup -} - -type Region struct { - ID string - VPCs []VPC -} - -type Instance interface { - GetID() string - GetRootVolume(ctx context.Context) (Volume, error) - WaitForReady(ctx context.Context) error - Delete(ctx context.Context) error -} - -type Volume interface { - TakeSnapshot(ctx context.Context) (Snapshot, error) -} - -type Snapshot interface { - GetID() string - GetRegion() string - Copy(ctx context.Context, dstRegion string) (Snapshot, error) - Delete(ctx context.Context) error - WaitForReady(ctx context.Context) error -} diff --git a/runtime_scan/pkg/scanner/job_managment.go b/runtime_scan/pkg/scanner/job_managment.go index a77581827..be3691172 100644 --- a/runtime_scan/pkg/scanner/job_managment.go +++ b/runtime_scan/pkg/scanner/job_managment.go @@ -24,10 +24,11 @@ import ( log "github.com/sirupsen/logrus" "github.com/openclarity/vmclarity/runtime_scan/pkg/config" - "github.com/openclarity/vmclarity/runtime_scan/pkg/provider" "github.com/openclarity/vmclarity/runtime_scan/pkg/types" ) +// TODO this code is taken from KubeClarity, we can make improvements base on the discussions here: https://github.com/openclarity/vmclarity/pull/3 + // run jobs. func (s *Scanner) jobBatchManagement(scanDone chan struct{}) { s.Lock() @@ -148,46 +149,52 @@ func (s *Scanner) waitForResult(data *scanData, ks chan bool) { } } -func (s *Scanner) runJob(ctx context.Context, data *scanData) (provider.Job, error) { +func (s *Scanner) runJob(ctx context.Context, data *scanData) (types.Job, error) { instanceToScan := data.instance + var launchSnapshot types.Snapshot + var cpySnapshot types.Snapshot volume, err := instanceToScan.GetRootVolume(ctx) if err != nil { - return provider.Job{}, fmt.Errorf("failed to get root volume of an instance %v: %v", instanceToScan.GetID(), err) + return types.Job{}, fmt.Errorf("failed to get root volume of an instance %v: %v", instanceToScan.GetID(), err) } snapshot, err := volume.TakeSnapshot(ctx) if err != nil { - return provider.Job{}, fmt.Errorf("failed to take snapshot of a volume: %v", err) + return types.Job{}, fmt.Errorf("failed to take snapshot of a volume: %v", err) } if err := snapshot.WaitForReady(ctx); err != nil { - return provider.Job{}, fmt.Errorf("failed to wait for snapshot %v ready: %v", snapshot.GetID(), err) + return types.Job{}, fmt.Errorf("failed to wait for snapshot %v ready: %v", snapshot.GetID(), err) } + launchSnapshot = snapshot - cpySnapshot, err := snapshot.Copy(ctx, s.region) - if err != nil { - return provider.Job{}, fmt.Errorf("failed to copy snapshot %v: %v", snapshot.GetID(), err) - } + if s.region != snapshot.GetRegion() { + cpySnapshot, err = snapshot.Copy(ctx, s.region) + if err != nil { + return types.Job{}, fmt.Errorf("failed to copy snapshot %v: %v", snapshot.GetID(), err) + } - if err := cpySnapshot.WaitForReady(ctx); err != nil { - return provider.Job{}, fmt.Errorf("failed wait for snapshot %v ready: %v", cpySnapshot.GetID(), err) + if err := cpySnapshot.WaitForReady(ctx); err != nil { + return types.Job{}, fmt.Errorf("failed wait for snapshot %v ready: %v", cpySnapshot.GetID(), err) + } + launchSnapshot = cpySnapshot } - i, err := s.providerClient.LaunchInstance(ctx, cpySnapshot) + i, err := s.providerClient.LaunchInstance(ctx, launchSnapshot) if err != nil { - return provider.Job{}, fmt.Errorf("failed to launch a new instance: %v", err) + return types.Job{}, fmt.Errorf("failed to launch a new instance: %v", err) } - return provider.Job{ + return types.Job{ Instance: i, SrcSnapshot: snapshot, DstSnapshot: cpySnapshot, }, nil } -func (s *Scanner) deleteJobIfNeeded(ctx context.Context, job *provider.Job, isSuccessfulJob, isCompletedJob bool) { +func (s *Scanner) deleteJobIfNeeded(ctx context.Context, job *types.Job, isSuccessfulJob, isCompletedJob bool) { if job == nil { return } @@ -210,7 +217,7 @@ func (s *Scanner) deleteJobIfNeeded(ctx context.Context, job *provider.Job, isSu } } -func (s *Scanner) deleteJob(ctx context.Context, job *provider.Job) { +func (s *Scanner) deleteJob(ctx context.Context, job *types.Job) { if err := job.Instance.Delete(ctx); err != nil { log.Errorf("failed to delete instance: %v", err) } diff --git a/runtime_scan/pkg/scanner/scanner.go b/runtime_scan/pkg/scanner/scanner.go index bb517fff2..62fdb8d3a 100644 --- a/runtime_scan/pkg/scanner/scanner.go +++ b/runtime_scan/pkg/scanner/scanner.go @@ -43,7 +43,7 @@ type Scanner struct { } type scanData struct { - instance provider.Instance + instance types.Instance scanUUID string vulnerabilitiesResult vulnerabilitiesScanResult resultChan chan bool diff --git a/runtime_scan/pkg/types/types.go b/runtime_scan/pkg/types/types.go index a74863049..1cd4eb21d 100644 --- a/runtime_scan/pkg/types/types.go +++ b/runtime_scan/pkg/types/types.go @@ -15,15 +15,7 @@ package types -import "github.com/openclarity/vmclarity/runtime_scan/pkg/provider" - -type ScanScope struct { - All bool - Regions []provider.Region - ScanStopped bool - IncludeTags []*provider.Tag - ExcludeTags []*provider.Tag -} +import "context" type Status string @@ -49,7 +41,7 @@ func (s *ScanProgress) SetStatus(status Status) { type InstanceScanResult struct { // Instance data - Instance provider.Instance + Instance Instance // Scan results Vulnerabilities []string // TODO define vulnerabilities struct Success bool @@ -60,3 +52,37 @@ type ScanResults struct { InstanceScanResults []*InstanceScanResult Progress ScanProgress } +type Job struct { + Instance Instance + SrcSnapshot Snapshot + DstSnapshot Snapshot +} + +type JobConfig struct { + InstanceToScan Instance + Region string + ImageID string + DeviceName string + SubnetID string +} + +type Instance interface { + GetID() string + GetRootVolume(ctx context.Context) (Volume, error) + WaitForReady(ctx context.Context) error + Delete(ctx context.Context) error +} + +type Volume interface { + TakeSnapshot(ctx context.Context) (Snapshot, error) +} + +type Snapshot interface { + GetID() string + GetRegion() string + Copy(ctx context.Context, dstRegion string) (Snapshot, error) + Delete(ctx context.Context) error + WaitForReady(ctx context.Context) error +} + + From 0cf422b94af06fca08d261377c910a6dc463c5d7 Mon Sep 17 00:00:00 2001 From: Erez Fishhimer Date: Sun, 20 Nov 2022 09:12:52 +0200 Subject: [PATCH 16/23] add license --- runtime_scan/pkg/provider/aws/types.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/runtime_scan/pkg/provider/aws/types.go b/runtime_scan/pkg/provider/aws/types.go index 61d0ad064..3dc086e8e 100644 --- a/runtime_scan/pkg/provider/aws/types.go +++ b/runtime_scan/pkg/provider/aws/types.go @@ -1,3 +1,18 @@ +// Copyright © 2022 Cisco Systems, Inc. and its affiliates. +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package aws type ScanScope struct { From f3f86106d58e002a7928bd7e7ddcf404f0c1a341 Mon Sep 17 00:00:00 2001 From: Erez Fishhimer Date: Sun, 20 Nov 2022 09:17:53 +0200 Subject: [PATCH 17/23] format --- runtime_scan/pkg/provider/aws/client_test.go | 4 ++-- runtime_scan/pkg/provider/aws/instance.go | 6 +++--- runtime_scan/pkg/types/errors.go | 2 +- runtime_scan/pkg/types/types.go | 2 -- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/runtime_scan/pkg/provider/aws/client_test.go b/runtime_scan/pkg/provider/aws/client_test.go index bf468ddea..1c144d02c 100644 --- a/runtime_scan/pkg/provider/aws/client_test.go +++ b/runtime_scan/pkg/provider/aws/client_test.go @@ -538,7 +538,7 @@ func Test_getInstanceState(t *testing.T) { } func TestClient_getInstancesFromDescribeInstancesOutput(t *testing.T) { - type fields struct {} + type fields struct{} type args struct { result *ec2.DescribeInstancesOutput excludeTags []Tag @@ -698,4 +698,4 @@ func TestClient_getInstancesFromDescribeInstancesOutput(t *testing.T) { } }) } -} \ No newline at end of file +} diff --git a/runtime_scan/pkg/provider/aws/instance.go b/runtime_scan/pkg/provider/aws/instance.go index d088dca87..abf0b60c4 100644 --- a/runtime_scan/pkg/provider/aws/instance.go +++ b/runtime_scan/pkg/provider/aws/instance.go @@ -28,9 +28,9 @@ import ( ) type InstanceImpl struct { - ec2Client *ec2.Client - id string - region string + ec2Client *ec2.Client + id string + region string } func (i *InstanceImpl) GetID() string { diff --git a/runtime_scan/pkg/types/errors.go b/runtime_scan/pkg/types/errors.go index 40396cc53..7fb8e37c4 100644 --- a/runtime_scan/pkg/types/errors.go +++ b/runtime_scan/pkg/types/errors.go @@ -31,5 +31,5 @@ const ( type ScanErrorSource string const ( - ScanErrSourceJob ScanErrorSource = "ScanErrSourceJob" + ScanErrSourceJob ScanErrorSource = "ScanErrSourceJob" ) diff --git a/runtime_scan/pkg/types/types.go b/runtime_scan/pkg/types/types.go index 1cd4eb21d..f63b77546 100644 --- a/runtime_scan/pkg/types/types.go +++ b/runtime_scan/pkg/types/types.go @@ -84,5 +84,3 @@ type Snapshot interface { Delete(ctx context.Context) error WaitForReady(ctx context.Context) error } - - From 30975bf09013fd8284f40a9acec81f9119917298 Mon Sep 17 00:00:00 2001 From: Erez Fishhimer Date: Sun, 20 Nov 2022 10:45:25 +0200 Subject: [PATCH 18/23] some changes --- runtime_scan/pkg/orchestrator/orchestrator.go | 2 +- runtime_scan/pkg/provider/aws/client.go | 14 ++++++++------ runtime_scan/pkg/provider/aws/client_test.go | 2 +- runtime_scan/pkg/provider/aws/instance.go | 6 +++--- runtime_scan/pkg/provider/aws/snapshot.go | 6 +++--- runtime_scan/pkg/provider/aws/types.go | 12 +++++++++++- runtime_scan/pkg/scanner/job_managment.go | 1 + runtime_scan/pkg/scanner/scanner.go | 2 +- 8 files changed, 29 insertions(+), 16 deletions(-) diff --git a/runtime_scan/pkg/orchestrator/orchestrator.go b/runtime_scan/pkg/orchestrator/orchestrator.go index 0dc3a4bac..572f41196 100644 --- a/runtime_scan/pkg/orchestrator/orchestrator.go +++ b/runtime_scan/pkg/orchestrator/orchestrator.go @@ -73,7 +73,7 @@ func (o *Orchestrator) Stop() { func (o *Orchestrator) Scan(scanConfig *_config.ScanConfig, scanDone chan struct{}) error { instances, err := o.providerClient.Discover(context.TODO(), &scanConfig.ScanScope) if err != nil { - return err + return fmt.Errorf("failed to discover instances to scan: %v", err) } scanConfig.Instances = instances diff --git a/runtime_scan/pkg/provider/aws/client.go b/runtime_scan/pkg/provider/aws/client.go index efbaa255d..e3eb50ae9 100644 --- a/runtime_scan/pkg/provider/aws/client.go +++ b/runtime_scan/pkg/provider/aws/client.go @@ -79,7 +79,7 @@ func (c *Client) Discover(ctx context.Context, scanScope interface{}) ([]types.I if len(regions) == 0 { return nil, fmt.Errorf("no regions to scan") } - filters = append(filters, createInclusionTagsFilters(scope.IncludeTags)...) + filters = append(filters, createInclusionTagsFilters(scope.TagSelector)...) filters = append(filters, createInstanceStateFilters(scope.ScanStopped)...) for _, region := range regions { @@ -150,11 +150,11 @@ func (c *Client) LaunchInstance(ctx context.Context, snapshot types.Snapshot) (t } func (c *Client) GetInstances(ctx context.Context, filters []ec2types.Filter, excludeTags []Tag, regionID string) ([]types.Instance, error) { - var ret []types.Instance + var ret = make([]types.Instance, 0) out, err := c.ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{ Filters: filters, - MaxResults: utils.Int32Ptr(50), // TODO what will be a good number? + MaxResults: utils.Int32Ptr(maxResults), // TODO what will be a good number? }, func(options *ec2.Options) { options.Region = regionID }) @@ -168,7 +168,7 @@ func (c *Client) GetInstances(ctx context.Context, filters []ec2types.Filter, ex for out.NextToken != nil { out, err = c.ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{ Filters: filters, - MaxResults: utils.Int32Ptr(50), + MaxResults: utils.Int32Ptr(maxResults), NextToken: out.NextToken, }, func(options *ec2.Options) { options.Region = regionID @@ -229,7 +229,7 @@ const ( func createVPCFilters(vpc VPC) []ec2types.Filter { var ret = make([]ec2types.Filter, 0) - var sgs []string + var sgs = make([]string, 0) // create per vpc filters ret = append(ret, ec2types.Filter{ @@ -250,7 +250,7 @@ func createVPCFilters(vpc VPC) []ec2types.Filter { } func createInstanceStateFilters(scanStopped bool) []ec2types.Filter { - var filters []ec2types.Filter + var filters = make([]ec2types.Filter, 0) var states = []string{"running"} if scanStopped { states = append(states, "stopped") @@ -268,6 +268,8 @@ func createInstanceStateFilters(scanStopped bool) []ec2types.Filter { func createInclusionTagsFilters(tags []Tag) []ec2types.Filter { var filters []ec2types.Filter + // If you specify multiple filters, the filters are joined with an AND, and the request returns + // only results that match all of the specified filters. for _, tag := range tags { filters = append(filters, ec2types.Filter{ Name: utils.StringPtr("tag:" + tag.key), diff --git a/runtime_scan/pkg/provider/aws/client_test.go b/runtime_scan/pkg/provider/aws/client_test.go index 1c144d02c..ac5986096 100644 --- a/runtime_scan/pkg/provider/aws/client_test.go +++ b/runtime_scan/pkg/provider/aws/client_test.go @@ -77,7 +77,7 @@ import ( // }, // }, // ScanStopped: true, -// IncludeTags: []*types.Tag{ +// TagSelector: []*types.Tag{ // { // key: "Name", // val: "diff-vpc", diff --git a/runtime_scan/pkg/provider/aws/instance.go b/runtime_scan/pkg/provider/aws/instance.go index abf0b60c4..c0d1025e5 100644 --- a/runtime_scan/pkg/provider/aws/instance.go +++ b/runtime_scan/pkg/provider/aws/instance.go @@ -78,11 +78,11 @@ func (i *InstanceImpl) GetRootVolume(ctx context.Context) (types.Volume, error) } func (i *InstanceImpl) WaitForReady(ctx context.Context) error { - ctxWithTimeout, _ := context.WithTimeout(context.Background(), 3*time.Minute) + ctxWithTimeout, _ := context.WithTimeout(context.Background(), waitTimeout*time.Minute) for { select { - case <-time.After(3 * time.Second): + case <-time.After(checkInterval * time.Second): out, err := i.ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{ InstanceIds: []string{i.id}, }, func(options *ec2.Options) { @@ -96,7 +96,7 @@ func (i *InstanceImpl) WaitForReady(ctx context.Context) error { return nil } case <-ctxWithTimeout.Done(): - return ctxWithTimeout.Err() + return fmt.Errorf("timeout: %v", ctxWithTimeout.Err()) } } } diff --git a/runtime_scan/pkg/provider/aws/snapshot.go b/runtime_scan/pkg/provider/aws/snapshot.go index 56e78a5df..fb2f93bba 100644 --- a/runtime_scan/pkg/provider/aws/snapshot.go +++ b/runtime_scan/pkg/provider/aws/snapshot.go @@ -83,11 +83,11 @@ func (s *SnapshotImpl) Delete(ctx context.Context) error { } func (s *SnapshotImpl) WaitForReady(ctx context.Context) error { - ctxWithTimeout, _ := context.WithTimeout(context.Background(), 3*time.Minute) + ctxWithTimeout, _ := context.WithTimeout(context.Background(), waitTimeout*time.Minute) for { select { - case <-time.After(3 * time.Second): + case <-time.After(checkInterval * time.Second): out, err := s.ec2Client.DescribeSnapshots(ctx, &ec2.DescribeSnapshotsInput{ SnapshotIds: []string{s.id}, }, func(options *ec2.Options) { @@ -103,7 +103,7 @@ func (s *SnapshotImpl) WaitForReady(ctx context.Context) error { return nil } case <-ctxWithTimeout.Done(): - return ctxWithTimeout.Err() + return fmt.Errorf("timeout: %v", ctxWithTimeout.Err()) } } } diff --git a/runtime_scan/pkg/provider/aws/types.go b/runtime_scan/pkg/provider/aws/types.go index 3dc086e8e..b76a88d95 100644 --- a/runtime_scan/pkg/provider/aws/types.go +++ b/runtime_scan/pkg/provider/aws/types.go @@ -15,11 +15,21 @@ package aws +const ( + waitTimeout = 3 + checkInterval = 3 + maxResults = 50 +) + type ScanScope struct { All bool Regions []Region ScanStopped bool - IncludeTags []Tag + // Only targets that have these tags will be selected for scanning within the selected scan scope. + // Multiple tags will be treated as an AND operator. + TagSelector []Tag + // Targets that have these tags will be excluded from the scan, even match the tag selector. + // Multiple tags will be treated as an AND operator. ExcludeTags []Tag } diff --git a/runtime_scan/pkg/scanner/job_managment.go b/runtime_scan/pkg/scanner/job_managment.go index be3691172..79cd2c8fa 100644 --- a/runtime_scan/pkg/scanner/job_managment.go +++ b/runtime_scan/pkg/scanner/job_managment.go @@ -149,6 +149,7 @@ func (s *Scanner) waitForResult(data *scanData, ks chan bool) { } } +// TODO need to clean all the created resources in case of a failure. func (s *Scanner) runJob(ctx context.Context, data *scanData) (types.Job, error) { instanceToScan := data.instance var launchSnapshot types.Snapshot diff --git a/runtime_scan/pkg/scanner/scanner.go b/runtime_scan/pkg/scanner/scanner.go index 62fdb8d3a..4e5511e9f 100644 --- a/runtime_scan/pkg/scanner/scanner.go +++ b/runtime_scan/pkg/scanner/scanner.go @@ -149,7 +149,7 @@ func (s *Scanner) Results() *types.ScanResults { s.Lock() defer s.Unlock() - var instanceScanResults []*types.InstanceScanResult + var instanceScanResults = make([]*types.InstanceScanResult, 0) for _, scanD := range s.instanceIDToScanData { if !scanD.completed { From d6f276bae710bb65ba51611d72c76299a6251884 Mon Sep 17 00:00:00 2001 From: Erez Fishhimer Date: Sun, 20 Nov 2022 11:01:52 +0200 Subject: [PATCH 19/23] exclude tags and logic --- runtime_scan/pkg/provider/aws/client.go | 30 +++++++++++------ runtime_scan/pkg/provider/aws/client_test.go | 34 ++++++++++++++++++-- runtime_scan/pkg/provider/aws/types.go | 2 +- 3 files changed, 54 insertions(+), 12 deletions(-) diff --git a/runtime_scan/pkg/provider/aws/client.go b/runtime_scan/pkg/provider/aws/client.go index e3eb50ae9..6804796d3 100644 --- a/runtime_scan/pkg/provider/aws/client.go +++ b/runtime_scan/pkg/provider/aws/client.go @@ -304,18 +304,30 @@ func (c *Client) ListAllRegions(ctx context.Context) ([]Region, error) { return ret, nil } +// AND logic - if excludeTags = {tag1:val1, tag2:val2}, +// then instance will be excluded only if he have ALL this tags ({tag1:val1, tag2:val2}) func hasExcludeTags(excludeTags []Tag, instanceTags []ec2types.Tag) bool { - var excludedTagsMap = make(map[string]string) + var instanceTagsMap = make(map[string]string) - for _, tag := range excludeTags { - excludedTagsMap[tag.key] = tag.val + if len(excludeTags) == 0 { + return false } - for _, instanceTag := range instanceTags { - if val, ok := excludedTagsMap[*instanceTag.Key]; ok { - if strings.Compare(val, *instanceTag.Value) == 0 { - return true - } + if len(instanceTags) == 0 { + return false + } + + for _, tag := range instanceTags { + instanceTagsMap[*tag.Key] = *tag.Value + } + + for _, tag := range excludeTags { + val, ok := instanceTagsMap[tag.key] + if !ok { + return false + } + if !(strings.Compare(val, tag.val) == 0) { + return false } } - return false + return true } diff --git a/runtime_scan/pkg/provider/aws/client_test.go b/runtime_scan/pkg/provider/aws/client_test.go index ac5986096..03f7faa36 100644 --- a/runtime_scan/pkg/provider/aws/client_test.go +++ b/runtime_scan/pkg/provider/aws/client_test.go @@ -314,7 +314,7 @@ func Test_hasExcludedTags(t *testing.T) { want: false, }, { - name: "instance has excluded tags", + name: "instance does not have ALL the excluded tags", args: args{ excludeTags: []Tag{ { @@ -337,10 +337,40 @@ func Test_hasExcludedTags(t *testing.T) { }, }, }, + want: false, + }, + { + name: "instance has ALL excluded tags", + args: args{ + excludeTags: []Tag{ + { + key: tagName1, + val: tagVal1, + }, + { + key: tagName2, + val: tagVal2, + }, + }, + instanceTags: []ec2types.Tag{ + { + Key: &tagName1, + Value: &tagVal1, + }, + { + Key: &tagName2, + Value: &tagVal2, + }, + { + Key: utils.StringPtr("stam"), + Value: utils.StringPtr("stam"), + }, + }, + }, want: true, }, { - name: "instance does not have excluded tags", + name: "instance does not have excluded tags at all", args: args{ excludeTags: []Tag{ { diff --git a/runtime_scan/pkg/provider/aws/types.go b/runtime_scan/pkg/provider/aws/types.go index b76a88d95..3fa480f89 100644 --- a/runtime_scan/pkg/provider/aws/types.go +++ b/runtime_scan/pkg/provider/aws/types.go @@ -28,7 +28,7 @@ type ScanScope struct { // Only targets that have these tags will be selected for scanning within the selected scan scope. // Multiple tags will be treated as an AND operator. TagSelector []Tag - // Targets that have these tags will be excluded from the scan, even match the tag selector. + // Targets that have these tags will be excluded from the scan, even if they match the tag selector. // Multiple tags will be treated as an AND operator. ExcludeTags []Tag } From aa9dc6dfd4cf3c26c9f9368c006c7be12c364a87 Mon Sep 17 00:00:00 2001 From: Erez Fishhimer Date: Sun, 20 Nov 2022 12:19:05 +0200 Subject: [PATCH 20/23] review --- runtime_scan/pkg/config/scan_config.go | 5 +- runtime_scan/pkg/provider/aws/client.go | 12 +-- runtime_scan/pkg/provider/aws/client_test.go | 103 +------------------ runtime_scan/pkg/provider/aws/types.go | 4 +- runtime_scan/pkg/provider/client.go | 2 +- runtime_scan/pkg/types/types.go | 2 + 6 files changed, 18 insertions(+), 110 deletions(-) diff --git a/runtime_scan/pkg/config/scan_config.go b/runtime_scan/pkg/config/scan_config.go index 7e62e595d..9bb579495 100644 --- a/runtime_scan/pkg/config/scan_config.go +++ b/runtime_scan/pkg/config/scan_config.go @@ -33,8 +33,9 @@ const ( type ScanConfig struct { MaxScanParallelism int // instances to scan - Instances []types.Instance - ScanScope interface{} + Instances []types.Instance + // per provider scan scope + ScanScope types.ScanScope JobResultTimeout time.Duration DeleteJobPolicy DeleteJobPolicyType } diff --git a/runtime_scan/pkg/provider/aws/client.go b/runtime_scan/pkg/provider/aws/client.go index 6804796d3..670b85f51 100644 --- a/runtime_scan/pkg/provider/aws/client.go +++ b/runtime_scan/pkg/provider/aws/client.go @@ -62,7 +62,7 @@ func Create(ctx context.Context, config *aws.Config) (*Client, error) { return &awsClient, nil } -func (c *Client) Discover(ctx context.Context, scanScope interface{}) ([]types.Instance, error) { +func (c *Client) Discover(ctx context.Context, scanScope types.ScanScope) ([]types.Instance, error) { var ret []types.Instance var filters []ec2types.Filter var scope *ScanScope @@ -85,7 +85,7 @@ func (c *Client) Discover(ctx context.Context, scanScope interface{}) ([]types.I for _, region := range regions { // if no vpcs, that mean that we don't need any vpc filters if len(region.vpcs) == 0 { - instances, err := c.GetInstances(ctx, filters, scope.ExcludeTags, region.Id) + instances, err := c.GetInstances(ctx, filters, scope.ExcludeTags, region.id) if err != nil { return nil, fmt.Errorf("failed to get instances: %v", err) } @@ -97,7 +97,7 @@ func (c *Client) Discover(ctx context.Context, scanScope interface{}) ([]types.I for _, vpc := range region.vpcs { vpcFilters := append(filters, createVPCFilters(vpc)...) - instances, err := c.GetInstances(ctx, vpcFilters, scope.ExcludeTags, region.Id) + instances, err := c.GetInstances(ctx, vpcFilters, scope.ExcludeTags, region.id) if err != nil { return nil, fmt.Errorf("failed to get instances: %v", err) } @@ -234,7 +234,7 @@ func createVPCFilters(vpc VPC) []ec2types.Filter { // create per vpc filters ret = append(ret, ec2types.Filter{ Name: utils.StringPtr(vpcIDFilterName), - Values: []string{vpc.Id}, + Values: []string{vpc.id}, }) sgs = getVPCSecurityGroupsIDs(vpc) if len(sgs) > 0 { @@ -298,14 +298,14 @@ func (c *Client) ListAllRegions(ctx context.Context) ([]Region, error) { } for _, region := range out.Regions { ret = append(ret, Region{ - Id: *region.RegionName, + id: *region.RegionName, }) } return ret, nil } // AND logic - if excludeTags = {tag1:val1, tag2:val2}, -// then instance will be excluded only if he have ALL this tags ({tag1:val1, tag2:val2}) +// then an instance will be excluded only if it has ALL these tags ({tag1:val1, tag2:val2}) func hasExcludeTags(excludeTags []Tag, instanceTags []ec2types.Tag) bool { var instanceTagsMap = make(map[string]string) diff --git a/runtime_scan/pkg/provider/aws/client_test.go b/runtime_scan/pkg/provider/aws/client_test.go index 03f7faa36..de49f646b 100644 --- a/runtime_scan/pkg/provider/aws/client_test.go +++ b/runtime_scan/pkg/provider/aws/client_test.go @@ -26,101 +26,6 @@ import ( "github.com/openclarity/vmclarity/runtime_scan/pkg/utils" ) -// -//func TestClient_ListAllRegions(t *testing.T) { -// //cfg, err := awsconfig.LoadDefaultConfig(context.TODO()) -// //if err != nil { -// // t.Fatalf("%v", err) -// //} -// //ec2Client := ec2.NewFromConfig(cfg) -// -// c, err := Create() -// if err != nil { -// t.Fatalf("%v", err) -// } -// //instance := types.Instance { -// // ID: "i-0f70b335ea12b2853", -// // Region: "us-east-1", -// //} -// scope := types.ScanScope{ -// All: false, -// Regions: []types.Region{ -// { -// ID: "us-east-2", -// vpcs: []types.VPC{ -// { -// ID: "vpc-32ea7c59", -// securityGroups: []types.SecurityGroup{ -// { -// ID: "sg-4d6b853a", -// }, -// }, -// }, -// }, -// }, -// { -// ID: "us-east-1", -// vpcs: []types.VPC{ -// { -// ID: "vpc-0c41450ba658eed00", -// securityGroups: nil, -// }, -// { -// ID: "vpc-9ca32ce1", -// securityGroups: []types.SecurityGroup{ -// { -// ID: "sg-030918e8e73254d42", -// }, -// }, -// }, -// }, -// }, -// }, -// ScanStopped: true, -// TagSelector: []*types.Tag{ -// { -// key: "Name", -// val: "diff-vpc", -// }, -// }, -// ExcludeTags: nil, -// } -// instances, err := c.Discover(&scope) -// if err != nil { -// t.Fatalf("%v", err) -// } -// -// instance := instances[0] -// -// rootVolume, err := c.GetInstanceRootVolume(instance) -// if err != nil { -// t.Fatalf("%v", err) -// } -// // create a snapshot of that vm -// srcSnapshot, err := c.CreateSnapshot(rootVolume) -// if err != nil { -// t.Fatalf("%v", err) -// } -// if err := c.WaitForSnapshotReady(srcSnapshot); err != nil { -// t.Fatalf("%v", err) -// } -// //copy the snapshot to the scanner region -// cpySnapshot, err := c.CopySnapshot(srcSnapshot, "us-east-2") -// if err != nil { -// t.Fatalf("%v", err) -// } -// if err := c.WaitForSnapshotReady(cpySnapshot); err != nil { -// t.Fatalf("%v", err) -// } -// // create the scanner job (vm) with a boot script -// launchedInstance, err := c.LaunchInstance("ami-0568773882d492fc8", "xvdh", cpySnapshot) -// if err != nil { -// t.Fatalf("%v", err) -// } -// -// t.Logf("res: %v", launchedInstance.ID) -//} - func Test_createVPCFilters(t *testing.T) { var ( vpcID = "vpc-1" @@ -143,7 +48,7 @@ func Test_createVPCFilters(t *testing.T) { name: "vpc with no security group", args: args{ vpc: VPC{ - Id: vpcID, + id: vpcID, securityGroups: nil, }, }, @@ -158,7 +63,7 @@ func Test_createVPCFilters(t *testing.T) { name: "vpc with one security group", args: args{ vpc: VPC{ - Id: vpcID, + id: vpcID, securityGroups: []SecurityGroup{ { id: sgID1, @@ -181,7 +86,7 @@ func Test_createVPCFilters(t *testing.T) { name: "vpc with two security groups", args: args{ vpc: VPC{ - Id: vpcID, + id: vpcID, securityGroups: []SecurityGroup{ { id: sgID1, @@ -314,7 +219,7 @@ func Test_hasExcludedTags(t *testing.T) { want: false, }, { - name: "instance does not have ALL the excluded tags", + name: "instance does not have ALL the excluded tags (partial matching)", args: args{ excludeTags: []Tag{ { diff --git a/runtime_scan/pkg/provider/aws/types.go b/runtime_scan/pkg/provider/aws/types.go index 3fa480f89..4f3864e59 100644 --- a/runtime_scan/pkg/provider/aws/types.go +++ b/runtime_scan/pkg/provider/aws/types.go @@ -43,11 +43,11 @@ type SecurityGroup struct { } type VPC struct { - Id string + id string securityGroups []SecurityGroup } type Region struct { - Id string + id string vpcs []VPC } diff --git a/runtime_scan/pkg/provider/client.go b/runtime_scan/pkg/provider/client.go index a01f4392c..7d8c9de48 100644 --- a/runtime_scan/pkg/provider/client.go +++ b/runtime_scan/pkg/provider/client.go @@ -23,7 +23,7 @@ import ( type Client interface { // Discover - list VM instances in the account according to the scan scope. - Discover(context.Context, interface{}) ([]types.Instance, error) + Discover(ctx context.Context, scanScope types.ScanScope) ([]types.Instance, error) // RunScanningJob - run a scanning job LaunchInstance(ctx context.Context, snapshot types.Snapshot) (types.Instance, error) } diff --git a/runtime_scan/pkg/types/types.go b/runtime_scan/pkg/types/types.go index f63b77546..570f77820 100644 --- a/runtime_scan/pkg/types/types.go +++ b/runtime_scan/pkg/types/types.go @@ -28,6 +28,8 @@ const ( DoneScanning Status = "DoneScanning" ) +type ScanScope interface {} + type ScanProgress struct { InstancesToScan uint32 InstancesStartedToScan uint32 From b6b1935bc5fd5d95fe442d4b8846c2b04feb1ef7 Mon Sep 17 00:00:00 2001 From: Erez Fishhimer Date: Sun, 20 Nov 2022 12:50:17 +0200 Subject: [PATCH 21/23] lint --- runtime_scan/pkg/orchestrator/orchestrator.go | 1 - runtime_scan/pkg/provider/aws/client.go | 14 +++++++------- runtime_scan/pkg/provider/aws/instance.go | 1 + runtime_scan/pkg/provider/aws/snapshot.go | 1 + runtime_scan/pkg/scanner/job_managment.go | 2 -- runtime_scan/pkg/scanner/scanner.go | 12 +++++------- runtime_scan/pkg/types/types.go | 2 +- 7 files changed, 15 insertions(+), 18 deletions(-) diff --git a/runtime_scan/pkg/orchestrator/orchestrator.go b/runtime_scan/pkg/orchestrator/orchestrator.go index 572f41196..50015b4f4 100644 --- a/runtime_scan/pkg/orchestrator/orchestrator.go +++ b/runtime_scan/pkg/orchestrator/orchestrator.go @@ -67,7 +67,6 @@ func (o *Orchestrator) Stop() { o.Clear() log.Infof("Stopping Orchestrator server") - //o.server.Stop() } func (o *Orchestrator) Scan(scanConfig *_config.ScanConfig, scanDone chan struct{}) error { diff --git a/runtime_scan/pkg/provider/aws/client.go b/runtime_scan/pkg/provider/aws/client.go index 670b85f51..74cb6f8d3 100644 --- a/runtime_scan/pkg/provider/aws/client.go +++ b/runtime_scan/pkg/provider/aws/client.go @@ -48,7 +48,7 @@ var ( ) func Create(ctx context.Context, config *aws.Config) (*Client, error) { - var awsClient = Client{ + var awsClient = Client { awsConfig: config, } @@ -57,6 +57,7 @@ func Create(ctx context.Context, config *aws.Config) (*Client, error) { return nil, fmt.Errorf("failed to load aws config: %v", err) } + // nolint:contextcheck awsClient.ec2Client = ec2.NewFromConfig(cfg) return &awsClient, nil @@ -214,7 +215,7 @@ func (c *Client) getInstancesFromDescribeInstancesOutput(result *ec2.DescribeIns } func getVPCSecurityGroupsIDs(vpc VPC) []string { - var sgs []string + var sgs = make([]string, len(vpc.securityGroups)) for _, sg := range vpc.securityGroups { sgs = append(sgs, sg.id) } @@ -229,14 +230,13 @@ const ( func createVPCFilters(vpc VPC) []ec2types.Filter { var ret = make([]ec2types.Filter, 0) - var sgs = make([]string, 0) // create per vpc filters ret = append(ret, ec2types.Filter{ Name: utils.StringPtr(vpcIDFilterName), Values: []string{vpc.id}, }) - sgs = getVPCSecurityGroupsIDs(vpc) + sgs := getVPCSecurityGroupsIDs(vpc) if len(sgs) > 0 { ret = append(ret, ec2types.Filter{ Name: utils.StringPtr(sgIDFilterName), @@ -266,7 +266,7 @@ func createInstanceStateFilters(scanStopped bool) []ec2types.Filter { } func createInclusionTagsFilters(tags []Tag) []ec2types.Filter { - var filters []ec2types.Filter + var filters = make([]ec2types.Filter, 0) // If you specify multiple filters, the filters are joined with an AND, and the request returns // only results that match all of the specified filters. @@ -289,7 +289,7 @@ func (c *Client) getRegionsToScan(ctx context.Context, scope *ScanScope) ([]Regi } func (c *Client) ListAllRegions(ctx context.Context) ([]Region, error) { - var ret []Region + var ret = make([]Region, 0) out, err := c.ec2Client.DescribeRegions(ctx, &ec2.DescribeRegionsInput{ AllRegions: nil, // display also disabled regions? }) @@ -305,7 +305,7 @@ func (c *Client) ListAllRegions(ctx context.Context) ([]Region, error) { } // AND logic - if excludeTags = {tag1:val1, tag2:val2}, -// then an instance will be excluded only if it has ALL these tags ({tag1:val1, tag2:val2}) +// then an instance will be excluded only if it has ALL these tags ({tag1:val1, tag2:val2}). func hasExcludeTags(excludeTags []Tag, instanceTags []ec2types.Tag) bool { var instanceTagsMap = make(map[string]string) diff --git a/runtime_scan/pkg/provider/aws/instance.go b/runtime_scan/pkg/provider/aws/instance.go index c0d1025e5..c3a60926e 100644 --- a/runtime_scan/pkg/provider/aws/instance.go +++ b/runtime_scan/pkg/provider/aws/instance.go @@ -78,6 +78,7 @@ func (i *InstanceImpl) GetRootVolume(ctx context.Context) (types.Volume, error) } func (i *InstanceImpl) WaitForReady(ctx context.Context) error { + // nolint:govet ctxWithTimeout, _ := context.WithTimeout(context.Background(), waitTimeout*time.Minute) for { diff --git a/runtime_scan/pkg/provider/aws/snapshot.go b/runtime_scan/pkg/provider/aws/snapshot.go index fb2f93bba..5aef677c0 100644 --- a/runtime_scan/pkg/provider/aws/snapshot.go +++ b/runtime_scan/pkg/provider/aws/snapshot.go @@ -83,6 +83,7 @@ func (s *SnapshotImpl) Delete(ctx context.Context) error { } func (s *SnapshotImpl) WaitForReady(ctx context.Context) error { + // nolint:govet ctxWithTimeout, _ := context.WithTimeout(context.Background(), waitTimeout*time.Minute) for { diff --git a/runtime_scan/pkg/scanner/job_managment.go b/runtime_scan/pkg/scanner/job_managment.go index 79cd2c8fa..e133339a5 100644 --- a/runtime_scan/pkg/scanner/job_managment.go +++ b/runtime_scan/pkg/scanner/job_managment.go @@ -163,11 +163,9 @@ func (s *Scanner) runJob(ctx context.Context, data *scanData) (types.Job, error) snapshot, err := volume.TakeSnapshot(ctx) if err != nil { return types.Job{}, fmt.Errorf("failed to take snapshot of a volume: %v", err) - } if err := snapshot.WaitForReady(ctx); err != nil { return types.Job{}, fmt.Errorf("failed to wait for snapshot %v ready: %v", snapshot.GetID(), err) - } launchSnapshot = snapshot diff --git a/runtime_scan/pkg/scanner/scanner.go b/runtime_scan/pkg/scanner/scanner.go index 4e5511e9f..04c6b1678 100644 --- a/runtime_scan/pkg/scanner/scanner.go +++ b/runtime_scan/pkg/scanner/scanner.go @@ -17,7 +17,6 @@ package scanner import ( "fmt" - "go/scanner" "sync" "sync/atomic" @@ -54,10 +53,10 @@ type scanData struct { } type vulnerabilitiesScanResult struct { - result []string - success bool - completed bool - error *scanner.Error + result []string + // success bool + // completed bool + // error *scanner.Error } func CreateScanner(config *_config.Config, providerClient provider.Client) *Scanner { @@ -76,7 +75,7 @@ func CreateScanner(config *_config.Config, providerClient provider.Client) *Scan } // initScan Calculate properties of scan targets -// nolint:cyclop +// nolint:cyclop,unparam func (s *Scanner) initScan() error { instanceIDToScanData := make(map[string]*scanData) @@ -159,7 +158,6 @@ func (s *Scanner) Results() *types.ScanResults { Instance: scanD.instance, Vulnerabilities: scanD.vulnerabilitiesResult.result, Success: scanD.success, - //ScanErrors: scanD.getScanErrors(), }) } diff --git a/runtime_scan/pkg/types/types.go b/runtime_scan/pkg/types/types.go index 570f77820..c30902941 100644 --- a/runtime_scan/pkg/types/types.go +++ b/runtime_scan/pkg/types/types.go @@ -28,7 +28,7 @@ const ( DoneScanning Status = "DoneScanning" ) -type ScanScope interface {} +type ScanScope interface{} type ScanProgress struct { InstancesToScan uint32 From 5dc3c09ceef93fe1df57642d5a72e8d638f24469 Mon Sep 17 00:00:00 2001 From: Erez Fishhimer Date: Sun, 20 Nov 2022 12:59:46 +0200 Subject: [PATCH 22/23] formt --- runtime_scan/pkg/orchestrator/orchestrator.go | 1 - runtime_scan/pkg/provider/aws/client.go | 18 +++++++++--------- runtime_scan/pkg/scanner/scanner.go | 2 +- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/runtime_scan/pkg/orchestrator/orchestrator.go b/runtime_scan/pkg/orchestrator/orchestrator.go index 50015b4f4..bb87ce82a 100644 --- a/runtime_scan/pkg/orchestrator/orchestrator.go +++ b/runtime_scan/pkg/orchestrator/orchestrator.go @@ -60,7 +60,6 @@ func Create(config *_config.Config, client provider.Client) (*Orchestrator, erro func (o *Orchestrator) Start(errChan chan struct{}) { // Start result server log.Infof("Starting Orchestrator server") - // o.server.Start(errChan) } func (o *Orchestrator) Stop() { diff --git a/runtime_scan/pkg/provider/aws/client.go b/runtime_scan/pkg/provider/aws/client.go index 74cb6f8d3..850aa63f3 100644 --- a/runtime_scan/pkg/provider/aws/client.go +++ b/runtime_scan/pkg/provider/aws/client.go @@ -48,7 +48,7 @@ var ( ) func Create(ctx context.Context, config *aws.Config) (*Client, error) { - var awsClient = Client { + awsClient := Client{ awsConfig: config, } @@ -151,7 +151,7 @@ func (c *Client) LaunchInstance(ctx context.Context, snapshot types.Snapshot) (t } func (c *Client) GetInstances(ctx context.Context, filters []ec2types.Filter, excludeTags []Tag, regionID string) ([]types.Instance, error) { - var ret = make([]types.Instance, 0) + ret := make([]types.Instance, 0) out, err := c.ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{ Filters: filters, @@ -215,7 +215,7 @@ func (c *Client) getInstancesFromDescribeInstancesOutput(result *ec2.DescribeIns } func getVPCSecurityGroupsIDs(vpc VPC) []string { - var sgs = make([]string, len(vpc.securityGroups)) + sgs := make([]string, len(vpc.securityGroups)) for _, sg := range vpc.securityGroups { sgs = append(sgs, sg.id) } @@ -229,7 +229,7 @@ const ( ) func createVPCFilters(vpc VPC) []ec2types.Filter { - var ret = make([]ec2types.Filter, 0) + ret := make([]ec2types.Filter, 0) // create per vpc filters ret = append(ret, ec2types.Filter{ @@ -250,8 +250,8 @@ func createVPCFilters(vpc VPC) []ec2types.Filter { } func createInstanceStateFilters(scanStopped bool) []ec2types.Filter { - var filters = make([]ec2types.Filter, 0) - var states = []string{"running"} + filters := make([]ec2types.Filter, 0) + states := []string{"running"} if scanStopped { states = append(states, "stopped") } @@ -266,7 +266,7 @@ func createInstanceStateFilters(scanStopped bool) []ec2types.Filter { } func createInclusionTagsFilters(tags []Tag) []ec2types.Filter { - var filters = make([]ec2types.Filter, 0) + filters := make([]ec2types.Filter, 0) // If you specify multiple filters, the filters are joined with an AND, and the request returns // only results that match all of the specified filters. @@ -289,7 +289,7 @@ func (c *Client) getRegionsToScan(ctx context.Context, scope *ScanScope) ([]Regi } func (c *Client) ListAllRegions(ctx context.Context) ([]Region, error) { - var ret = make([]Region, 0) + ret := make([]Region, 0) out, err := c.ec2Client.DescribeRegions(ctx, &ec2.DescribeRegionsInput{ AllRegions: nil, // display also disabled regions? }) @@ -307,7 +307,7 @@ func (c *Client) ListAllRegions(ctx context.Context) ([]Region, error) { // AND logic - if excludeTags = {tag1:val1, tag2:val2}, // then an instance will be excluded only if it has ALL these tags ({tag1:val1, tag2:val2}). func hasExcludeTags(excludeTags []Tag, instanceTags []ec2types.Tag) bool { - var instanceTagsMap = make(map[string]string) + instanceTagsMap := make(map[string]string) if len(excludeTags) == 0 { return false diff --git a/runtime_scan/pkg/scanner/scanner.go b/runtime_scan/pkg/scanner/scanner.go index 04c6b1678..bb7a47568 100644 --- a/runtime_scan/pkg/scanner/scanner.go +++ b/runtime_scan/pkg/scanner/scanner.go @@ -148,7 +148,7 @@ func (s *Scanner) Results() *types.ScanResults { s.Lock() defer s.Unlock() - var instanceScanResults = make([]*types.InstanceScanResult, 0) + instanceScanResults := make([]*types.InstanceScanResult, 0) for _, scanD := range s.instanceIDToScanData { if !scanD.completed { From 2504140e10e8b6acbcc58007f82c9b565d7ed0d6 Mon Sep 17 00:00:00 2001 From: Erez Fishhimer Date: Sun, 20 Nov 2022 13:05:59 +0200 Subject: [PATCH 23/23] lint --- runtime_scan/pkg/orchestrator/orchestrator.go | 2 +- runtime_scan/pkg/provider/aws/client.go | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/runtime_scan/pkg/orchestrator/orchestrator.go b/runtime_scan/pkg/orchestrator/orchestrator.go index bb87ce82a..43cd58e38 100644 --- a/runtime_scan/pkg/orchestrator/orchestrator.go +++ b/runtime_scan/pkg/orchestrator/orchestrator.go @@ -89,7 +89,7 @@ func (o *Orchestrator) ScanProgress() types.ScanProgress { func (o *Orchestrator) Results() *types.ScanResults { return nil // TODO - //return o.getScanner().Results() + // return o.getScanner().Results() } func (o *Orchestrator) Clear() { diff --git a/runtime_scan/pkg/provider/aws/client.go b/runtime_scan/pkg/provider/aws/client.go index 850aa63f3..fa4c5da92 100644 --- a/runtime_scan/pkg/provider/aws/client.go +++ b/runtime_scan/pkg/provider/aws/client.go @@ -216,8 +216,8 @@ func (c *Client) getInstancesFromDescribeInstancesOutput(result *ec2.DescribeIns func getVPCSecurityGroupsIDs(vpc VPC) []string { sgs := make([]string, len(vpc.securityGroups)) - for _, sg := range vpc.securityGroups { - sgs = append(sgs, sg.id) + for i, sg := range vpc.securityGroups { + sgs[i] = sg.id } return sgs } @@ -266,7 +266,8 @@ func createInstanceStateFilters(scanStopped bool) []ec2types.Filter { } func createInclusionTagsFilters(tags []Tag) []ec2types.Filter { - filters := make([]ec2types.Filter, 0) + // nolint:prealloc + var filters []ec2types.Filter // If you specify multiple filters, the filters are joined with an AND, and the request returns // only results that match all of the specified filters.