diff --git a/deepfence_agent/plugins/cloud-scanner b/deepfence_agent/plugins/cloud-scanner index 239c1f42f9..c25aedcfdb 160000 --- a/deepfence_agent/plugins/cloud-scanner +++ b/deepfence_agent/plugins/cloud-scanner @@ -1 +1 @@ -Subproject commit 239c1f42f92c3a783fdb5b7e622b6d75611e47ab +Subproject commit c25aedcfdbc09a9deae9f40cdea9367a6b6ca658 diff --git a/deepfence_agent/plugins/package-scanner b/deepfence_agent/plugins/package-scanner index d89caea19d..0de7189a74 160000 --- a/deepfence_agent/plugins/package-scanner +++ b/deepfence_agent/plugins/package-scanner @@ -1 +1 @@ -Subproject commit d89caea19d4d4b5a9c87d97d9f6f9dd51567a0b8 +Subproject commit 0de7189a7425b30dce4961e4ba6745932dbcbf4d diff --git a/deepfence_agent/plugins/yara-rules b/deepfence_agent/plugins/yara-rules index 140a38dfb7..8217b51893 160000 --- a/deepfence_agent/plugins/yara-rules +++ b/deepfence_agent/plugins/yara-rules @@ -1 +1 @@ -Subproject commit 140a38dfb7c110759010a281ccb6eb3d79e910cb +Subproject commit 8217b518934b556ee7f56e6e5fc3e05be8c8d9fa diff --git a/deepfence_bootstrapper/assets/config.ini b/deepfence_bootstrapper/assets/config.ini index b6ff6aea5c..ae066802ad 100644 --- a/deepfence_bootstrapper/assets/config.ini +++ b/deepfence_bootstrapper/assets/config.ini @@ -21,13 +21,13 @@ autostart=true autorestart=true [process:secret_scanner] -command=/bin/bash -c "rm -f $DF_INSTALL_DIR/tmp/secret-scanner.sock && exec $DF_INSTALL_DIR/home/deepfence/bin/secret-scanner/SecretScanner --config-path $DF_INSTALL_DIR/home/deepfence/bin/secret-scanner/config --rules-path $DF_INSTALL_DIR/home/deepfence/bin/secret-scanner/rules/secret-yara-rules --socket-path=$DF_INSTALL_DIR/tmp/secret-scanner.sock" +command=/bin/bash -c "rm -f $DF_INSTALL_DIR/tmp/secret-scanner.sock && exec $DF_INSTALL_DIR/home/deepfence/bin/secret-scanner/SecretScanner --config-path $DF_INSTALL_DIR/home/deepfence/bin/secret-scanner/config --rules-path $DF_INSTALL_DIR/home/deepfence/bin/secret-scanner/rules --socket-path=$DF_INSTALL_DIR/tmp/secret-scanner.sock" path=$DF_INSTALL_DIR/home/deepfence/bin/secret-scanner/SecretScanner autostart=true autorestart=true [process:malware_scanner] -command=/bin/bash -c "rm -f $DF_INSTALL_DIR/tmp/yara-hunter.sock && exec $DF_INSTALL_DIR/home/deepfence/bin/yara-hunter/YaraHunter --config-path $DF_INSTALL_DIR/home/deepfence/bin/yara-hunter/config.yaml --rules-path $DF_INSTALL_DIR/home/deepfence/bin/yara-hunter/yara-rules --socket-path=$DF_INSTALL_DIR/tmp/yara-hunter.sock --enable-updater=false" +command=/bin/bash -c "rm -f $DF_INSTALL_DIR/tmp/yara-hunter.sock && exec $DF_INSTALL_DIR/home/deepfence/bin/yara-hunter/YaraHunter --config-path $DF_INSTALL_DIR/home/deepfence/bin/yara-hunter/config.yaml --rules-path $DF_INSTALL_DIR/home/deepfence/bin/yara-hunter/rules --socket-path=$DF_INSTALL_DIR/tmp/yara-hunter.sock --enable-updater=false" path=$DF_INSTALL_DIR/home/deepfence/bin/yara-hunter/YaraHunter autostart=true autorestart=true diff --git a/deepfence_bootstrapper/go.mod b/deepfence_bootstrapper/go.mod index d815ab67d6..9e50423e3f 100644 --- a/deepfence_bootstrapper/go.mod +++ b/deepfence_bootstrapper/go.mod @@ -34,6 +34,8 @@ require ( require ( aead.dev/minisign v0.2.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect + github.com/VirusTotal/gyp v0.9.0 // indirect + github.com/XSAM/otelsql v0.31.0 // indirect github.com/c9s/goprocinfo v0.0.0-20151025191153-19cb9f127a9c // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect @@ -44,6 +46,12 @@ require ( github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/docker/docker v1.4.2-0.20180827131323-0c5f8d2b9b23 // indirect github.com/docker/go-units v0.4.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/glebarez/go-sqlite v1.21.2 // indirect + github.com/glebarez/sqlite v1.11.0 // indirect + github.com/go-chi/jwtauth/v5 v5.3.1 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/godbus/dbus/v5 v5.0.4 // indirect github.com/golang/protobuf v1.5.4 // indirect @@ -52,8 +60,12 @@ require ( github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.7.5 // indirect github.com/hibiken/asynq v0.24.1 // indirect + github.com/jellydator/ttlcache/v3 v3.2.0 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect github.com/k-sone/critbitgo v1.2.0 // indirect github.com/klauspost/compress v1.17.8 // indirect + github.com/klauspost/cpuid/v2 v2.2.6 // indirect github.com/kr/pty v1.1.1 // indirect github.com/lestrrat-go/blackmagic v1.0.2 // indirect github.com/lestrrat-go/httpcc v1.0.1 // indirect @@ -64,10 +76,16 @@ require ( github.com/lib/pq v1.10.9 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/minio-go/v7 v7.0.70 // indirect + github.com/neo4j/neo4j-go-driver/v5 v5.20.0 // indirect github.com/opentracing/opentracing-go v1.1.0 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect + github.com/raito-io/neo4j-tracing v0.0.5 // indirect github.com/redis/go-redis/v9 v9.5.1 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/robfig/cron/v3 v3.0.1 // indirect + github.com/rs/xid v1.5.0 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/spf13/cast v1.6.0 // indirect @@ -78,12 +96,21 @@ require ( github.com/weaveworks/common v0.0.0-20200310113808-2708ba4e60a4 // indirect github.com/weaveworks/ps v0.0.0-20160725183535-70d17b2d6f76 // indirect github.com/willdonnelly/passwd v0.0.0-20141013001024-7935dab3074c // indirect + go.opentelemetry.io/otel v1.26.0 // indirect + go.opentelemetry.io/otel/metric v1.26.0 // indirect + go.opentelemetry.io/otel/trace v1.26.0 // indirect golang.org/x/crypto v0.22.0 // indirect golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2 // indirect golang.org/x/net v0.24.0 // indirect + golang.org/x/sync v0.1.0 // indirect golang.org/x/sys v0.20.0 // indirect golang.org/x/text v0.15.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect google.golang.org/protobuf v1.34.0 // indirect + gorm.io/gorm v1.25.11 // indirect + modernc.org/libc v1.22.5 // indirect + modernc.org/mathutil v1.5.0 // indirect + modernc.org/memory v1.5.0 // indirect + modernc.org/sqlite v1.23.1 // indirect ) diff --git a/deepfence_bootstrapper/go.sum b/deepfence_bootstrapper/go.sum index 4f8f2f1556..22efaff1c2 100644 --- a/deepfence_bootstrapper/go.sum +++ b/deepfence_bootstrapper/go.sum @@ -9,6 +9,10 @@ github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3 github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= github.com/NYTimes/gziphandler v1.0.2-0.20180227021810-5032c8878b9d/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= +github.com/VirusTotal/gyp v0.9.0 h1:jhOBl93jfStmAcKLa/EcTmdPng5bn5kvJJZqQqJ5R4g= +github.com/VirusTotal/gyp v0.9.0/go.mod h1:nmcW15dQ1657PmMcG9X/EZmp6rTQsyo9g8r6Cz1/AHc= +github.com/XSAM/otelsql v0.31.0 h1:AcWI+/BW4ANKyAybZmU9g9kjjSIcDEOFw96ybyM4cDo= +github.com/XSAM/otelsql v0.31.0/go.mod h1:iCkLyB/me+QC4yjymXjLimJiX0oklymiKeGxeGDTW24= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -69,6 +73,8 @@ github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/libnetwork v0.8.0-dev.2.0.20180608203834-19279f049241/go.mod h1:93m0aTqz6z+g32wla4l4WxTrdtvBRmVzYRkYvasA5Z8= github.com/dustin/go-humanize v0.0.0-20160923163517-bd88f87ad3a4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/evanphx/json-patch v0.0.0-20170719203123-944e07253867/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fluent/fluent-logger-golang v1.5.0/go.mod h1:2/HCT/jTy78yGyeNGQLGQsjF3zzzAuy6Xlk6FCMV5eU= @@ -76,11 +82,24 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsouza/go-dockerclient v1.3.0/go.mod h1:IN9UPc4/w7cXiARH2Yg99XxUHbAM+6rAi9hzBVbkWRU= +github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9gAXWo= +github.com/glebarez/go-sqlite v1.21.2/go.mod h1:sfxdZyhQjTM2Wry3gVYWaW072Ri1WMdWJi0k6+3382k= +github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw= +github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ= +github.com/go-chi/chi/v5 v5.0.7 h1:rDTPXLDHGATaeHvVlLcR4Qe0zftYethFucbjVQ1PxU8= +github.com/go-chi/chi/v5 v5.0.7/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/go-chi/jwtauth/v5 v5.3.1 h1:1ePWrjVctvp1tyBq5b/2ER8Th/+RbYc7x4qNsc5rh5A= +github.com/go-chi/jwtauth/v5 v5.3.1/go.mod h1:6Fl2RRmWXs3tJYE1IQGX81FsPoGqDwq9c15j52R5q80= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= @@ -110,10 +129,13 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gopacket v1.1.17/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM= +github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= +github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -146,6 +168,12 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/iovisor/gobpf v0.0.0-20180826141936-4ece6c56f936/go.mod h1:+5U5qu5UOu8YJ5oHVLvWKH7/Dr5QNHU7mZ2RfPEeXg8= +github.com/jellydator/ttlcache/v3 v3.2.0 h1:6lqVJ8X3ZaUwvzENqPAobDsXNExfUJd61u++uW8a3LE= +github.com/jellydator/ttlcache/v3 v3.2.0/go.mod h1:hi7MGFdMAwZna5n2tuvh63DvFLzVKySzCVW6+0gA2n4= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -157,6 +185,9 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= +github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -192,6 +223,10 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v0.0.0-20160129163459-3d66e3747d22/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.70 h1:1u9NtMgfK1U42kUxcsl5v0yj6TEOPR497OAQxpJnn2g= +github.com/minio/minio-go/v7 v7.0.70/go.mod h1:4yBA8v80xGA30cfM3fz0DKYMXunWl/AV/6tWEs9ryzo= github.com/minio/selfupdate v0.6.0 h1:i76PgT0K5xO9+hjzKcacQtO7+MjJ4JKA8Ak8XQ9DDwU= github.com/minio/selfupdate v0.6.0/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM= github.com/mjibson/esc v0.2.0/go.mod h1:9Hw9gxxfHulMF5OJKCyhYD7PzlSdhzXyaGEBRPH1OPs= @@ -203,6 +238,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/nats-io/gnatsd v0.8.1-0.20160607194326-f2c17eb159e1/go.mod h1:nqco77VO78hLCJpIcVfygDP2rPGfsEHkGTUk94uh5DQ= github.com/nats-io/nats v1.2.1-0.20160607194537-ce9cdc9addff/go.mod h1:PpmYZwlgTfBI56QypJLfIMOfLnMRuVs+VL6r8mQ2SoQ= github.com/nats-io/nuid v0.0.0-20160402145409-a5152d67cf63/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/neo4j/neo4j-go-driver/v5 v5.20.0 h1:XnoAi6g6XRkX+wxWa3yM+f7PT2VUkGQfBGtGuJL4fsM= +github.com/neo4j/neo4j-go-driver/v5 v5.20.0/go.mod h1:Vff8OwT7QpLm7L2yYr85XNWe9Rbqlbeb9asNXJTHO4k= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= @@ -245,14 +282,20 @@ github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/raito-io/neo4j-tracing v0.0.5 h1:Z1eEQl1Imm0DFkR2yfMc7jVW+ix4oZxAVOAWPQBgI2Q= +github.com/raito-io/neo4j-tracing v0.0.5/go.mod h1:m0utJXW1BPoBdKZ1cVhpyVZ1ChWttj8pSVtka/5j63s= github.com/redis/go-redis/v9 v9.0.3/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= github.com/redis/go-redis/v9 v9.5.1 h1:H1X4D3yHPaYrkL5X06Wh6xNVM/pX0Ft4RV0vMGvLBh8= github.com/redis/go-redis/v9 v9.5.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/richo/GOSHOUT v0.0.0-20210103052837-9a2e452d4c18/go.mod h1:MSTsYcO3SGF1j/eewqZORAzbp3BUbisi2094EDP3+To= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.32.0 h1:keLypqrlIjaFsbmJOBdB/qvyF8KEtCWHwobLp5l/mQ0= github.com/rs/zerolog v1.32.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= @@ -316,9 +359,20 @@ github.com/weaveworks/weave v2.3.1-0.20180427133448-4da998ab4507+incompatible/go github.com/willdonnelly/passwd v0.0.0-20141013001024-7935dab3074c h1:4+NVyrLUuEmvE3r3Xst7gPuKhAP6X04ACpMmPvtK0M0= github.com/willdonnelly/passwd v0.0.0-20141013001024-7935dab3074c/go.mod h1:xcvfY9pOw6s4wyrhilFSbMthL6KzgrfCIETHHUOQ/fQ= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs= +go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4= +go.opentelemetry.io/otel/metric v1.26.0 h1:7S39CLuY5Jgg9CrnA9HHiEjGMF/X2VHvoXGgSllRz30= +go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4= +go.opentelemetry.io/otel/sdk v1.26.0 h1:Y7bumHf5tAiDlRYFmGqetNcLaVUZmh4iYfmGxtmz7F8= +go.opentelemetry.io/otel/sdk v1.26.0/go.mod h1:0p8MXpqLeJ0pzcszQQN4F0S5FVjBLgypeGSngLsmirs= +go.opentelemetry.io/otel/sdk/metric v1.26.0 h1:cWSks5tfriHPdWFnl+qpX3P681aAYqlZHcAyHw5aU9Y= +go.opentelemetry.io/otel/sdk/metric v1.26.0/go.mod h1:ClMFFknnThJCksebJwz7KIyEDHO+nTB6gK8obLy8RyE= +go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA= +go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180820150726-614d502a4dac/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -357,6 +411,8 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180824143301-4910a1d54f87/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -376,6 +432,7 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= @@ -416,6 +473,7 @@ google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ= google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.34.0 h1:Qo/qEd2RZPCf2nKuorzksSknv0d3ERwp1vFG38gSmH4= google.golang.org/protobuf v1.34.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= @@ -433,9 +491,12 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/gorm v1.25.11 h1:/Wfyg1B/je1hnDx3sMkX+gAlxrlZpn6X0BXRlwXlvHg= +gorm.io/gorm v1.25.11/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= gotest.tools v2.1.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= @@ -449,4 +510,12 @@ k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/kube-openapi v0.0.0-20180108222231-a07b7bbb58e7/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= k8s.io/utils v0.0.0-20200414100711-2df71ebbae66/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE= +modernc.org/libc v1.22.5/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY= +modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= +modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds= +modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/sqlite v1.23.1 h1:nrSBg4aRQQwq59JpvGEQ15tNxoO5pX/kUjcRNwSAGQM= +modernc.org/sqlite v1.23.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/deepfence_bootstrapper/router/malware_scanner.go b/deepfence_bootstrapper/router/malware_scanner.go index bb5c4bab62..459b9ac3cf 100644 --- a/deepfence_bootstrapper/router/malware_scanner.go +++ b/deepfence_bootstrapper/router/malware_scanner.go @@ -16,6 +16,7 @@ import ( "github.com/deepfence/ThreatMapper/deepfence_bootstrapper/supervisor" ctl "github.com/deepfence/ThreatMapper/deepfence_utils/controls" "github.com/deepfence/ThreatMapper/deepfence_utils/log" + "github.com/deepfence/ThreatMapper/deepfence_utils/threatintel" "github.com/deepfence/ThreatMapper/deepfence_utils/utils" ) @@ -139,10 +140,10 @@ func UpdateMalwareRules(req ctl.ThreatIntelInfo) error { } newRules := "new_malware_rules.tar.gz" - rulesPath := path.Join(dfUtils.GetDfInstallDir(), "/home/deepfence/bin/yara-hunter/yara-rules") + rulesPath := path.Join(dfUtils.GetDfInstallDir(), "/home/deepfence/bin/yara-hunter/rules") if err := downloadFile(newRules, req.MalwareRulesURL); err != nil { - log.Error().Err(err).Msg("failed to downlaod malware rules") + log.Error().Err(err).Msg("failed to download malware rules") return err } defer os.Remove(newRules) @@ -168,6 +169,15 @@ func UpdateMalwareRules(req ctl.ThreatIntelInfo) error { return err } + for _, infile := range []string{ + filepath.Join(rulesPath, "df-malware.json"), + } { + err = threatintel.ExtractDFRules2NativeRules(infile, rulesPath) + if err != nil { + return err + } + } + log.Info().Msg("malware rules updated starting malware scanner") // start scanner diff --git a/deepfence_bootstrapper/router/secret_scanner.go b/deepfence_bootstrapper/router/secret_scanner.go index be6f3f9a5f..454e0bfb5c 100644 --- a/deepfence_bootstrapper/router/secret_scanner.go +++ b/deepfence_bootstrapper/router/secret_scanner.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path" + "path/filepath" pb "github.com/deepfence/agent-plugins-grpc/srcgo" "google.golang.org/grpc" @@ -14,6 +15,7 @@ import ( "github.com/deepfence/ThreatMapper/deepfence_bootstrapper/supervisor" ctl "github.com/deepfence/ThreatMapper/deepfence_utils/controls" "github.com/deepfence/ThreatMapper/deepfence_utils/log" + "github.com/deepfence/ThreatMapper/deepfence_utils/threatintel" "github.com/deepfence/ThreatMapper/deepfence_utils/utils" dfUtils "github.com/deepfence/df-utils" ) @@ -126,7 +128,8 @@ func UpdateSecretsRules(req ctl.ThreatIntelInfo) error { } newRules := "new_secret_rules.tar.gz" - rulesPath := path.Join(dfUtils.GetDfInstallDir(), "/home/deepfence/bin/secret-scanner/rules") + rulesPkgPath := path.Join(dfUtils.GetDfInstallDir(), "/home/deepfence/bin/secret-scanner") + rulesPath := path.Join(rulesPkgPath, "rules") if err := downloadFile(newRules, req.SecretsRulesURL); err != nil { log.Error().Err(err).Msg("failed to download secrets rules") @@ -150,11 +153,20 @@ func UpdateSecretsRules(req ctl.ThreatIntelInfo) error { return err } - if err := utils.ExtractTarGz(bytes.NewReader(data), rulesPath); err != nil { + if err := utils.ExtractTarGz(bytes.NewReader(data), rulesPkgPath); err != nil { log.Error().Err(err).Msg("failed to extract rules") return err } + for _, infile := range []string{ + filepath.Join(rulesPath, "df-secret.json"), + } { + err = threatintel.ExtractDFRules2NativeRules(infile, rulesPath) + if err != nil { + return err + } + } + log.Info().Msg("secrets rules updated starting secret scanner") // start scanner diff --git a/deepfence_server/apiDocs/docs.go b/deepfence_server/apiDocs/docs.go index 6faea73e17..b6173bfd25 100644 --- a/deepfence_server/apiDocs/docs.go +++ b/deepfence_server/apiDocs/docs.go @@ -23,6 +23,7 @@ const ( tagSearch = "Search" tagThreat = "Threat" tagScanResults = "Scan Results" + tagRules = "Rules" tagSecretScan = "Secret Scan" tagVulnerability = "Vulnerability" tagMalwareScan = "Malware Scan" diff --git a/deepfence_server/apiDocs/operation.go b/deepfence_server/apiDocs/operation.go index e2a0c18842..b1471e3d1a 100644 --- a/deepfence_server/apiDocs/operation.go +++ b/deepfence_server/apiDocs/operation.go @@ -180,6 +180,14 @@ func (d *OpenAPIDocs) AddLookupOperations() { d.AddOperation("getComplianceControls", http.MethodPost, "/deepfence/lookup/compliance-controls", "Retrieve Cloud Compliances Control data", "Retrieve all the data associated with cloud compliance controls", http.StatusOK, []string{tagLookup}, bearerToken, new(LookupFilter), new([]CloudComplianceControl)) + + d.AddOperation("getMalwareRules", http.MethodPost, "/deepfence/lookup/malware-rules", + "Get Malware Rules", "Retrieve malware rule resources", + http.StatusOK, []string{tagLookup}, bearerToken, new(LookupFilter), new([]MalwareRule)) + + d.AddOperation("getSecretRules", http.MethodPost, "/deepfence/lookup/secret-rules", + "Get Secret Rules", "Retrieve secret rule resources", + http.StatusOK, []string{tagLookup}, bearerToken, new(LookupFilter), new([]SecretRule)) } func (d *OpenAPIDocs) AddSearchOperations() { @@ -639,6 +647,14 @@ func (d *OpenAPIDocs) AddScansOperations() { "Notify Scans Results", "Notify scan results in connected integration channels", http.StatusNoContent, []string{tagScanResults}, bearerToken, new(ScanResultsActionRequest), nil) + //Rules operations + d.AddOperation("unmaskRules", http.MethodPost, "/deepfence/rules/action/unmask", + "Unmask Rules", "Unmask rules", + http.StatusNoContent, []string{tagRules}, bearerToken, new(RulesActionRequest), nil) + d.AddOperation("maskRules", http.MethodPost, "/deepfence/rules/action/mask", + "mask Rules", "mask rules", + http.StatusNoContent, []string{tagRules}, bearerToken, new(RulesActionRequest), nil) + // Bulk Delete Scans d.AddOperation("bulkDeleteScans", http.MethodPost, "/deepfence/scans/bulk/delete", "Bulk Delete Scans", "Bulk delete scans along with their results for a particular scan type", diff --git a/deepfence_server/go.mod b/deepfence_server/go.mod index c1f5fe3d33..004f55ccb6 100644 --- a/deepfence_server/go.mod +++ b/deepfence_server/go.mod @@ -58,6 +58,7 @@ require ( ) require ( + github.com/VirusTotal/gyp v0.9.0 // indirect github.com/containerd/log v0.1.0 // indirect github.com/glebarez/go-sqlite v1.21.2 // indirect github.com/glebarez/sqlite v1.11.0 // indirect diff --git a/deepfence_server/go.sum b/deepfence_server/go.sum index 227f49524a..80a89c12f8 100644 --- a/deepfence_server/go.sum +++ b/deepfence_server/go.sum @@ -12,6 +12,8 @@ github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+q github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/PagerDuty/go-pagerduty v1.7.0 h1:S1NcMKECxT5hJwV4VT+QzeSsSiv4oWl1s2821dUqG/8= github.com/PagerDuty/go-pagerduty v1.7.0/go.mod h1:PuFyJKRz1liIAH4h5KVXVD18Obpp1ZXRdxHvmGXooro= +github.com/VirusTotal/gyp v0.9.0 h1:jhOBl93jfStmAcKLa/EcTmdPng5bn5kvJJZqQqJ5R4g= +github.com/VirusTotal/gyp v0.9.0/go.mod h1:nmcW15dQ1657PmMcG9X/EZmp6rTQsyo9g8r6Cz1/AHc= github.com/XSAM/otelsql v0.31.0 h1:AcWI+/BW4ANKyAybZmU9g9kjjSIcDEOFw96ybyM4cDo= github.com/XSAM/otelsql v0.31.0/go.mod h1:iCkLyB/me+QC4yjymXjLimJiX0oklymiKeGxeGDTW24= github.com/andygrunwald/go-jira v1.16.0 h1:PU7C7Fkk5L96JvPc6vDVIrd99vdPnYudHu4ju2c2ikQ= @@ -306,6 +308,7 @@ github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -467,6 +470,7 @@ google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.34.0 h1:Qo/qEd2RZPCf2nKuorzksSknv0d3ERwp1vFG38gSmH4= google.golang.org/protobuf v1.34.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/deepfence_server/handler/lookup_reports.go b/deepfence_server/handler/lookup_reports.go index 6e9fc60bd4..a3e11d71fb 100644 --- a/deepfence_server/handler/lookup_reports.go +++ b/deepfence_server/handler/lookup_reports.go @@ -129,3 +129,49 @@ func (h *Handler) GetCloudCompliances(w http.ResponseWriter, r *http.Request) { func (h *Handler) GetCloudComplianceControl(w http.ResponseWriter, r *http.Request) { getGeneric[model.CloudComplianceControl](h, w, r, reporters_lookup.GetCloudComplianceControl) } + +func (h *Handler) GetMalwareRules(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + var req reporters_lookup.LookupFilter + err := httpext.DecodeJSON(r, httpext.NoQueryParams, MaxPostRequestSize, &req) + if err != nil { + log.Error().Msg(err.Error()) + http.Error(w, "Error processing request body", http.StatusBadRequest) + return + } + + hosts, err := reporters_lookup.GetMalwareRulesReport(r.Context(), req) + if err != nil { + log.Error().Msg(err.Error()) + http.Error(w, "Error processing request body", http.StatusBadRequest) + return + } + + err = httpext.JSON(w, http.StatusOK, hosts) + if err != nil { + log.Error().Msg(err.Error()) + } +} + +func (h *Handler) GetSecretRules(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + var req reporters_lookup.LookupFilter + err := httpext.DecodeJSON(r, httpext.NoQueryParams, MaxPostRequestSize, &req) + if err != nil { + log.Error().Msg(err.Error()) + http.Error(w, "Error processing request body", http.StatusBadRequest) + return + } + + hosts, err := reporters_lookup.GetSecretRulesReport(r.Context(), req) + if err != nil { + log.Error().Msg(err.Error()) + http.Error(w, "Error processing request body", http.StatusBadRequest) + return + } + + err = httpext.JSON(w, http.StatusOK, hosts) + if err != nil { + log.Error().Msg(err.Error()) + } +} diff --git a/deepfence_server/handler/scan_reports.go b/deepfence_server/handler/scan_reports.go index d887eb3f6d..071c4cb4d5 100644 --- a/deepfence_server/handler/scan_reports.go +++ b/deepfence_server/handler/scan_reports.go @@ -39,6 +39,8 @@ import ( const ( MaxSbomRequestSize = 500 * 1e6 DownloadReportURLExpiry = 5 * time.Minute + actionMask = "mask" + actionUnMask = "unmask" ) var ( @@ -2460,3 +2462,61 @@ func startMultiComplianceScan(ctx context.Context, reqs []model.NodeIdentifier, } return scanIDs, bulkID, nil } + +func UpdateRulesMasked(ctx context.Context, req model.RulesActionRequest, value bool) error { + driver, err := directory.Neo4jClient(ctx) + if err != nil { + return err + } + session := driver.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeWrite}) + defer session.Close(ctx) + + tx, err := session.BeginTransaction(ctx, neo4j.WithTxTimeout(30*time.Second)) + if err != nil { + return err + } + defer tx.Close(ctx) + + _, err = tx.Run(ctx, ` + MATCH (n:DeepfenceRule) + WHERE n.rule_id IN $rule_ids + SET n.masked = $value`, map[string]interface{}{"rule_ids": req.RulesIDs, "value": value}) + if err != nil { + return err + } + return tx.Commit(ctx) +} + +func (h *Handler) RulesMaskHandler(w http.ResponseWriter, r *http.Request) { + h.rulesActionHandler(w, r, actionMask) +} + +func (h *Handler) RulesUnMaskHandler(w http.ResponseWriter, r *http.Request) { + h.rulesActionHandler(w, r, actionUnMask) +} + +func (h *Handler) rulesActionHandler(w http.ResponseWriter, r *http.Request, action string) { + defer r.Body.Close() + var req model.RulesActionRequest + err := httpext.DecodeJSON(r, httpext.NoQueryParams, MaxPostRequestSize, &req) + if err != nil { + h.respondError(err, w) + return + } + err = h.Validator.Struct(req) + if err != nil { + h.respondError(&ValidatorError{err: err}, w) + return + } + switch action { + case actionMask: + err = UpdateRulesMasked(r.Context(), req, true) + case actionUnMask: + err = UpdateRulesMasked(r.Context(), req, false) + } + if err != nil { + h.respondError(err, w) + return + } + w.WriteHeader(http.StatusNoContent) +} diff --git a/deepfence_server/model/lookup.go b/deepfence_server/model/lookup.go index 97d2d0d090..7ae9c3506d 100644 --- a/deepfence_server/model/lookup.go +++ b/deepfence_server/model/lookup.go @@ -435,3 +435,7 @@ func (CloudNode) ExtendedField() string { func (cr CloudNode) id() string { return cr.ID } + +type RulesActionRequest struct { + RulesIDs []string `json:"rule_ids" validate:"required,gt=0,dive,min=1" required:"true"` +} diff --git a/deepfence_server/model/scans.go b/deepfence_server/model/scans.go index b3d713f7c2..305e9a3bb9 100644 --- a/deepfence_server/model/scans.go +++ b/deepfence_server/model/scans.go @@ -325,12 +325,15 @@ func (Secret) GetJSONCategory() string { } type SecretRule struct { - ID int `json:"id"` + RuleID int `json:"rule_id"` Name string `json:"name"` Part string `json:"part"` SignatureToMatch string `json:"signature_to_match"` Level string `json:"level" required:"true"` Masked bool `json:"masked" required:"true"` + Summary string `json:"summary" required:"true"` + Payload string `json:"payload" required:"true"` + Severity string `json:"severity" required:"true"` UpdatedAt int64 `json:"updated_at" required:"true"` } @@ -351,27 +354,28 @@ func (SecretRule) GetJSONCategory() string { } type Vulnerability struct { - NodeID string `json:"node_id" required:"true"` - CveID string `json:"cve_id" required:"true"` - CveSeverity string `json:"cve_severity" validate:"required,oneof=critical high medium low unknown" required:"true" enum:"critical,high,medium,low,unknown"` - CveCausedByPackage string `json:"cve_caused_by_package" required:"true"` - CveCausedByPackagePath string `json:"cve_caused_by_package_path" required:"true"` - CveContainerLayer string `json:"cve_container_layer" required:"true"` - CveLink string `json:"cve_link" required:"true"` - Masked bool `json:"masked" required:"true"` - UpdatedAt int64 `json:"updated_at" required:"true"` - HasLiveConnection bool `json:"has_live_connection" required:"true"` - CveType string `json:"cve_type" required:"true"` - CveFixedIn string `json:"cve_fixed_in" required:"true"` - CveDescription string `json:"cve_description" required:"true"` - CveCVSSScore float64 `json:"cve_cvss_score" required:"true"` - CveOverallScore float64 `json:"cve_overall_score" required:"true"` - CveAttackVector string `json:"cve_attack_vector" required:"true"` - URLs []interface{} `json:"urls" required:"true"` - ExploitPOC string `json:"exploit_poc" required:"true"` - ParsedAttackVector string `json:"parsed_attack_vector" required:"true"` - Resources []BasicNode `json:"resources" required:"false"` - RuleID string `json:"rule_id" required:"true"` + NodeID string `json:"node_id" required:"true"` + CveID string `json:"cve_id" required:"true"` + CveSeverity string `json:"cve_severity" validate:"required,oneof=critical high medium low unknown" required:"true" enum:"critical,high,medium,low,unknown"` + CveCausedByPackage string `json:"cve_caused_by_package" required:"true"` + CveCausedByPackagePath string `json:"cve_caused_by_package_path" required:"true"` + CveContainerLayer string `json:"cve_container_layer" required:"true"` + CveLink string `json:"cve_link" required:"true"` + Masked bool `json:"masked" required:"true"` + UpdatedAt int64 `json:"updated_at" required:"true"` + HasLiveConnection bool `json:"has_live_connection" required:"true"` + CveType string `json:"cve_type" required:"true"` + CveFixedIn string `json:"cve_fixed_in" required:"true"` + CveDescription string `json:"cve_description" required:"true"` + CveCVSSScore float64 `json:"cve_cvss_score" required:"true"` + CveOverallScore float64 `json:"cve_overall_score" required:"true"` + CveAttackVector string `json:"cve_attack_vector" required:"true"` + URL string `json:"url" required:"true"` + ExploitPOC string `json:"exploit_poc" required:"true"` + ParsedAttackVector string `json:"parsed_attack_vector" required:"true"` + Resources []BasicNode `json:"resources" required:"false"` + RuleID string `json:"rule_id" required:"true"` + Namespace string `json:"namespace" required:"true"` } func (Vulnerability) NodeType() string { @@ -391,26 +395,26 @@ func (Vulnerability) GetJSONCategory() string { } type VulnerabilityRule struct { - NodeID string `json:"node_id" required:"true"` - CveID string `json:"cve_id" required:"true"` - CveType string `json:"cve_type" required:"true"` - CveSeverity string `json:"cve_severity" required:"true"` - CveFixedIn string `json:"cve_fixed_in" required:"true"` - CveLink string `json:"cve_link" required:"true"` - CveDescription string `json:"cve_description" required:"true"` - CveCVSSScore float64 `json:"cve_cvss_score" required:"true"` - CveOverallScore float64 `json:"cve_overall_score" required:"true"` - CveAttackVector string `json:"cve_attack_vector" required:"true"` - URLs []interface{} `json:"urls" required:"true"` - ExploitPOC string `json:"exploit_poc" required:"true"` - Masked bool `json:"masked" required:"true"` - UpdatedAt int64 `json:"updated_at" required:"true"` - PackageName string `json:"package_name"` - ParsedAttackVector string `json:"parsed_attack_vector" required:"true"` - Resources []BasicNode `json:"resources" required:"false"` - CISAKEV bool `json:"cisa_kev" required:"false"` - EPSSScore float64 `json:"epss_score" required:"false"` - Namespace string `json:"namespace" required:"false"` + NodeID string `json:"node_id" required:"true"` + CveID string `json:"cve_id" required:"true"` + CveTypes []string `json:"cve_types" required:"true"` + CveSeverities []string `json:"cve_severities" required:"true"` + CveFixedIns []string `json:"cve_fixed_ins" required:"true"` + CveLinks []string `json:"cve_links" required:"true"` + CveDescriptions []string `json:"cve_descriptions" required:"true"` + CveCvssScores []float64 `json:"cve_cvss_scores" required:"true"` + CveOverallScores []float64 `json:"cve_overall_scores" required:"true"` + CveAttackVectors []string `json:"cve_attack_vectors" required:"true"` + URLs []string `json:"urls" required:"true"` + ExploitPOCs []string `json:"exploit_pocs" required:"true"` + Masked bool `json:"masked" required:"true"` + UpdatedAt int64 `json:"updated_at" required:"true"` + PackageNames []string `json:"package_names" required:"true"` + ParsedAttackVectors []string `json:"parsed_attack_vectors" required:"true"` + Resources []BasicNode `json:"resources" required:"false"` + CISAKEV bool `json:"cisa_kev" required:"true"` + EPSSScore float64 `json:"epss_score" required:"true"` + Namespaces []string `json:"namespaces" required:"true"` } func (VulnerabilityRule) NodeType() string { @@ -422,7 +426,10 @@ func (VulnerabilityRule) ExtendedField() string { } func (v VulnerabilityRule) GetCategory() string { - return v.CveSeverity + if len(v.CveSeverities) > 0 { + return v.CveSeverities[0] + } + return "" } func (VulnerabilityRule) GetJSONCategory() string { @@ -480,6 +487,9 @@ type MalwareRule struct { Reference string `json:"reference"` FileSeverity string `json:"file_severity"` Masked bool `json:"masked" required:"true"` + Payload string `json:"payload" required:"true"` + Severity string `json:"severity" required:"true"` + Summary string `json:"summary" required:"true"` UpdatedAt int64 `json:"updated_at" required:"true"` } diff --git a/deepfence_server/reporters/lookup/lookup.go b/deepfence_server/reporters/lookup/lookup.go index c2cacb90de..f5ddc49cdd 100644 --- a/deepfence_server/reporters/lookup/lookup.go +++ b/deepfence_server/reporters/lookup/lookup.go @@ -710,3 +710,124 @@ func GetCloudComplianceReport(ctx context.Context, filter LookupFilter) ([]model func GetCloudComplianceControl(ctx context.Context, filter LookupFilter) ([]model.CloudComplianceControl, error) { return getGenericNodeReport[model.CloudComplianceControl](ctx, "control_id", filter) } + +func GetMalwareRulesReport(ctx context.Context, filter LookupFilter) ([]model.MalwareRule, error) { + rules, err := getGenericDirectRuleReport[model.MalwareRule](ctx, filter) + if err != nil { + return nil, err + } + return rules, nil +} + +func GetSecretRulesReport(ctx context.Context, filter LookupFilter) ([]model.SecretRule, error) { + rules, err := getGenericDirectRuleReport[model.SecretRule](ctx, filter) + if err != nil { + return nil, err + } + return rules, nil +} + +func getGenericDirectRuleReport[T reporters.Cypherable](ctx context.Context, filter LookupFilter) ([]T, error) { + + ctx, span := telemetry.NewSpan(ctx, "lookup", "get-generic-direct-rule-report") + defer span.End() + + res := []T{} + var dummy T + + driver, err := directory.Neo4jClient(ctx) + if err != nil { + return res, err + } + + session := driver.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeRead}) + defer session.Close(ctx) + + tx, err := session.BeginTransaction(ctx, neo4j.WithTxTimeout(30*time.Second)) + if err != nil { + return res, err + } + defer tx.Close(ctx) + + var r neo4j.ResultWithContext + query := ` + MATCH (n:` + dummy.NodeType() + `) + WHERE n.rule_id IN $ids + OPTIONAL MATCH (n) -[:IS]-> (e) + CALL { + WITH n + OPTIONAL MATCH (l) -[:DETECTED]-> (n) + OPTIONAL MATCH (l) -[:SCANNED]-> (k) + WITH distinct k + WHERE k.active=true + RETURN collect(coalesce(k.node_id, '') + '##' + coalesce(k.node_name, '') + '##' + coalesce(k.node_type, '')) as resources + } + RETURN ` + reporters.FieldFilterCypher("n", filter.InFieldFilter) + `, e, resources` + + log.Debug().Msgf("query: %s", query) + + r, err = tx.Run(ctx, query, map[string]interface{}{"ids": filter.NodeIds}) + if err != nil { + return res, err + } + + recs, err := r.Collect(ctx) + + if err != nil { + return res, err + } + + for _, rec := range recs { + var nodeMap map[string]interface{} + if len(filter.InFieldFilter) == 0 { + data, has := rec.Get("n") + if !has { + log.Warn().Msgf("Missing neo4j entry") + continue + } + da, ok := data.(dbtype.Node) + if !ok { + log.Warn().Msgf("Missing neo4j entry") + continue + } + nodeMap = da.Props + } else { + nodeMap = map[string]interface{}{} + for i := range filter.InFieldFilter { + nodeMap[filter.InFieldFilter[i]] = rec.Values[i] + } + } + isNode, _ := rec.Get("e") + if isNode != nil { + for k, v := range isNode.(dbtype.Node).Props { + if k != "node_id" { + nodeMap[k] = v + } else { + nodeMap[dummy.ExtendedField()] = v + } + } + } + resources, isValue := rec.Get("resources") + if isValue { + resourceList := resources.([]interface{}) + resourceListString := make([]model.BasicNode, len(resourceList)) + for i, v := range resourceList { + nodeDetails := strings.Split(v.(string), nodeReportResourcesSplit) + if len(nodeDetails) != 3 { + continue + } + resourceListString[i] = model.BasicNode{ + NodeID: nodeDetails[0], + Name: nodeDetails[1], + NodeType: nodeDetails[2], + } + } + nodeMap["resources"] = resourceListString + } + var node T + utils.FromMap(nodeMap, &node) + res = append(res, node) + } + + return res, nil +} diff --git a/deepfence_server/router/router.go b/deepfence_server/router/router.go index e0fe2098cf..656bcc67b9 100644 --- a/deepfence_server/router/router.go +++ b/deepfence_server/router/router.go @@ -259,6 +259,8 @@ func SetupRoutes(r *chi.Mux, serverPort string, serveOpenapiDocs bool, ingestC c r.Post("/compliances", dfHandler.GetCompliances) r.Post("/cloud-compliances", dfHandler.GetCloudCompliances) r.Post("/compliance-controls", dfHandler.GetCloudComplianceControl) + r.Post("/secret-rules", dfHandler.GetSecretRules) + r.Post("/malware-rules", dfHandler.GetMalwareRules) }) r.Route("/complete", func(r chi.Router) { @@ -457,6 +459,13 @@ func SetupRoutes(r *chi.Mux, serverPort string, serveOpenapiDocs bool, ingestC c r.Post("/notify", dfHandler.AuthHandler(ResourceScanReport, PermissionRead, dfHandler.ScanResultNotifyHandler)) }) + r.Route("/rules", func(r chi.Router) { + r.Route("/action", func(r chi.Router) { + r.Post("/mask", dfHandler.AuthHandler(ResourceScanReport, PermissionWrite, dfHandler.RulesMaskHandler)) + r.Post("/unmask", dfHandler.AuthHandler(ResourceScanReport, PermissionWrite, dfHandler.RulesUnMaskHandler)) + }) + }) + r.Post("/scans/bulk/delete", dfHandler.AuthHandler(ResourceScanReport, PermissionDelete, dfHandler.BulkDeleteScans)) r.Route("/scan/{scan_type}/{scan_id}", func(r chi.Router) { diff --git a/deepfence_utils/threatintel/feeds.go b/deepfence_utils/threatintel/feeds.go index fc88d650d4..e7400fa58e 100644 --- a/deepfence_utils/threatintel/feeds.go +++ b/deepfence_utils/threatintel/feeds.go @@ -1,5 +1,18 @@ package threatintel +import ( + "encoding/base64" + "encoding/json" + "fmt" + "io/fs" + "log" + "os" + "path/filepath" + "strings" + + "github.com/VirusTotal/gyp" +) + type Artefact struct { Name string `json:"name"` Type string `json:"type"` @@ -7,9 +20,11 @@ type Artefact struct { } type DeepfenceRule struct { - RuleID string `json:"rule_id"` - Type string `json:"type"` - Payload string `json:"payload"` + RuleID string `json:"rule_id"` + Type string `json:"type"` + Payload string `json:"payload"` + Severity string `json:"severity"` + Description string `json:"description"` } type TracerFeeds struct { @@ -32,6 +47,7 @@ type FeedsBundle struct { CreatedAt int64 `json:"created_at"` ScannerFeeds ScannerFeeds `json:"scanner_feeds"` TracerFeeds TracerFeeds `json:"tracer_feeds"` + Extra []string `json:"extra"` } func NewFeeds(createdAt int64, version string) *FeedsBundle { @@ -86,3 +102,82 @@ func (fb *FeedsBundle) AddComplianceRules(df []DeepfenceRule) { func (fb *FeedsBundle) AddCloudComplianceRules(df []DeepfenceRule) { fb.ScannerFeeds.CloudComplianceRules = append(fb.ScannerFeeds.CloudComplianceRules, df...) } + +func ExtractDFRules2NativeRules(inpath, outdir string) error { + var feeds FeedsBundle + inFile, err := os.OpenFile(inpath, os.O_RDONLY, fs.ModePerm) + if err != nil { + return err + } + defer inFile.Close() + + dec := json.NewDecoder(inFile) + err = dec.Decode(&feeds) + if err != nil { + return err + } + + if len(feeds.ScannerFeeds.MalwareRules) > 0 { + ExportYaraRules(outdir, feeds.ScannerFeeds.MalwareRules, feeds.Extra) + } + if len(feeds.ScannerFeeds.SecretRules) > 0 { + ExportYaraRules(outdir, feeds.ScannerFeeds.SecretRules, feeds.Extra) + } + + return nil +} + +func groupType2filenames(rules []DeepfenceRule) map[string][]DeepfenceRule { + res := map[string][]DeepfenceRule{} + for i := range rules { + index := strings.Index(rules[i].Type, "-") + filename := rules[i].Type + if index != -1 { + filename = rules[i].Type[index+1:] + } + + switch { + case strings.HasPrefix(rules[i].Type, "suricata"): + filename += ".rules" + case strings.HasPrefix(rules[i].Type, "modsec"): + filename += ".conf" + case strings.Contains(rules[i].Type, "secret"): + filename += ".secret.yar" + case strings.Contains(rules[i].Type, "malware"): + filename += ".malware.yar" + } + res[filename] = append(res[filename], rules[i]) + } + return res +} + +func ExportYaraRules(outDir string, rules []DeepfenceRule, extra []string) { + ruleGroups := groupType2filenames(rules) + + for k, groupRules := range ruleGroups { + file, err := os.OpenFile(filepath.Join(outDir, k), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, fs.ModePerm) + if err != nil { + log.Printf("failed to open file: %s, skipping", err) + continue + } + defer file.Close() + for i := range extra { + file.WriteString(fmt.Sprintf("import \"%s\"\n", extra[i])) + } + for _, rule := range groupRules { + decoded, err := base64.StdEncoding.DecodeString(rule.Payload) + if err != nil { + fmt.Printf("err on base 64: %v\n", err) + continue + } + rs, err := gyp.ParseString(string(decoded)) + if err != nil { + fmt.Printf("err on marshal: %v\n", err) + continue + } + for _, r := range rs.Rules { + r.WriteSource(file) + } + } + } +} diff --git a/deepfence_utils/threatintel/malware.go b/deepfence_utils/threatintel/malware.go index abef6e5c49..dcf5e31c78 100644 --- a/deepfence_utils/threatintel/malware.go +++ b/deepfence_utils/threatintel/malware.go @@ -1,8 +1,11 @@ package threatintel import ( + "archive/tar" "context" + "encoding/json" "fmt" + "io" "strings" "github.com/deepfence/ThreatMapper/deepfence_utils/directory" @@ -43,7 +46,13 @@ func DownloadMalwareRules(ctx context.Context, entry Entry) error { path, sha, err := UploadToMinio(ctx, content.Bytes(), MalwareRulesStore, fmt.Sprintf("malware-rules-%d.tar.gz", entry.Built.Unix())) if err != nil { - log.Error().Err(err).Msg("failed to uplaod malware rules to fileserver") + log.Error().Err(err).Msg("failed to upload malware rules to fileserver") + return err + } + + err = IngestMalwareRules(ctx, content.Bytes()) + if err != nil { + log.Error().Err(err).Msg("failed to ingest malware rules") return err } @@ -118,3 +127,48 @@ func FetchMalwareRulesInfo(ctx context.Context) (path, hash string, err error) { return rec.Values[0].(string), rec.Values[1].(string), nil } + +func IngestMalwareRules(ctx context.Context, content []byte) error { + nc, err := directory.Neo4jClient(ctx) + if err != nil { + return err + } + session := nc.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeWrite}) + defer session.Close(ctx) + + err = ProcessTarGz(content, func(header *tar.Header, reader io.Reader) error { + var feeds FeedsBundle + if header.FileInfo().IsDir() { + return nil + } + if strings.HasSuffix(header.Name, ".data") { + return nil + } + jdec := json.NewDecoder(reader) + err = jdec.Decode(&feeds) + if err != nil { + log.Warn().Msg(err.Error()) + return nil + } + + log.Info().Msgf("Ingesting %d malware", len(feeds.ScannerFeeds.MalwareRules)) + + _, err = session.Run(ctx, ` + UNWIND $rules as row + MERGE (n:DeepfenceRule:MalwareRule{rule_id: row.rule_id}) ++ SET n.type = row.type, ++ n.payload = row.payload, ++ n.summary = row.description, ++ n.severity = row.severity, ++ n.updated_at = TIMESTAMP()`, + map[string]interface{}{ + "rules": DeepfenceRule2json(feeds.ScannerFeeds.MalwareRules), + }) + return err + }) + if err != nil { + return err + } + + return err +} diff --git a/deepfence_utils/threatintel/secrets.go b/deepfence_utils/threatintel/secrets.go index e4b8d751db..147e470f24 100644 --- a/deepfence_utils/threatintel/secrets.go +++ b/deepfence_utils/threatintel/secrets.go @@ -1,8 +1,11 @@ package threatintel import ( + "archive/tar" "context" + "encoding/json" "fmt" + "io" "strings" "github.com/deepfence/ThreatMapper/deepfence_utils/directory" @@ -43,7 +46,13 @@ func DownloadSecretsRules(ctx context.Context, entry Entry) error { path, sha, err := UploadToMinio(ctx, content.Bytes(), SecretsRulesStore, fmt.Sprintf("secrets-rules-%d.tar.gz", entry.Built.Unix())) if err != nil { - log.Error().Err(err).Msg("failed to uplaod secrets rules to fileserver") + log.Error().Err(err).Msg("failed to upload secrets rules to fileserver") + return err + } + + err = IngestSecretRules(ctx, content.Bytes()) + if err != nil { + log.Error().Err(err).Msg("failed to ingest secrets rules") return err } @@ -118,3 +127,48 @@ func FetchSecretsRulesInfo(ctx context.Context) (path, hash string, err error) { return rec.Values[0].(string), rec.Values[1].(string), nil } + +func IngestSecretRules(ctx context.Context, content []byte) error { + nc, err := directory.Neo4jClient(ctx) + if err != nil { + return err + } + session := nc.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeWrite}) + defer session.Close(ctx) + + err = ProcessTarGz(content, func(header *tar.Header, reader io.Reader) error { + var feeds FeedsBundle + if header.FileInfo().IsDir() { + return nil + } + if strings.HasSuffix(header.Name, ".data") { + return nil + } + jdec := json.NewDecoder(reader) + err = jdec.Decode(&feeds) + if err != nil { + log.Warn().Msg(err.Error()) + return nil + } + + log.Info().Msgf("Ingesting %d secrets", len(feeds.ScannerFeeds.SecretRules)) + + _, err = session.Run(ctx, ` + UNWIND $rules as row + MERGE (n:DeepfenceRule:SecretRule{rule_id: row.rule_id}) + SET n.type = row.type, + n.payload = row.payload, + n.summary = row.description, + n.severity = row.severity, + n.updated_at = TIMESTAMP()`, + map[string]interface{}{ + "rules": DeepfenceRule2json(feeds.ScannerFeeds.SecretRules), + }) + return err + }) + if err != nil { + return err + } + + return err +} diff --git a/deepfence_utils/threatintel/vulndb.go b/deepfence_utils/threatintel/vulndb.go index 9e0289136f..26dc1ea22f 100644 --- a/deepfence_utils/threatintel/vulndb.go +++ b/deepfence_utils/threatintel/vulndb.go @@ -442,7 +442,7 @@ func ingestVulnerabilityRules(ctx context.Context, vulnerabilityDBModel vulnerab } vulnerabilityMetadataMap[v.ID+":"+v.Namespace] = meta } - vulnerabilityRules := make([]map[string]interface{}, 0) + vulnerabilityRules := make(map[string]ingesters.VulnerabilityRule) for _, v := range vulnerabilityDBModel.VulnerabilityModel { vulnerabilityMetadata, ok := vulnerabilityMetadataMap[v.ID+":"+v.Namespace] if !ok { @@ -502,35 +502,59 @@ func ingestVulnerabilityRules(ctx context.Context, vulnerabilityDBModel vulnerab cveType = ScanTypeBase } - vulnerabilityRule := ingesters.VulnerabilityRule{ - CveID: vulnerability.ID, - CveType: cveType, - CveSeverity: strings.ToLower(vulnerabilityMetadata.Severity), - CveFixedIn: cveFixedInVersion, - CveLink: vulnerabilityMetadata.DataSource, - CveDescription: vulnerabilityMetadata.Description, - CveCvssScore: cvssScore, - CveOverallScore: overallScore, - CveAttackVector: attackVector, - URLs: urls, - ExploitPOC: metasploitURL, - PackageName: vulnerability.PackageName, - ParsedAttackVector: parsedAttackVector, - CISAKEV: vulnerabilityMetadata.CISAKEV, - EPSSScore: vulnerabilityMetadata.EPSSScore, - Namespace: vulnerability.Namespace, + var vulnerabilityRule ingesters.VulnerabilityRule + if vulnerabilityRule, ok = vulnerabilityRules[vulnerability.ID]; !ok { + vulnerabilityRule = ingesters.VulnerabilityRule{ + // Common fields + CveID: vulnerability.ID, + CISAKEV: vulnerabilityMetadata.CISAKEV, + EPSSScore: vulnerabilityMetadata.EPSSScore, + } } - vulnerabilityRule.SetNodeID() - vulnerabilityRules = append(vulnerabilityRules, vulnerabilityRule.ToMap()) - if len(vulnerabilityRules) == neo4jWriteBatchSize { - _ = saveVulnerabilityRulesInNeo4j(ctx, vulnerabilityRules) - vulnerabilityRules = make([]map[string]interface{}, 0) + // Ordered insert + vulnerabilityRule.Namespaces = append(vulnerabilityRule.Namespaces, vulnerability.Namespace) + vulnerabilityRule.PackageNames = append(vulnerabilityRule.PackageNames, vulnerability.PackageName) + vulnerabilityRule.CveTypes = append(vulnerabilityRule.CveTypes, cveType) + vulnerabilityRule.CveSeverities = append(vulnerabilityRule.CveSeverities, strings.ToLower(vulnerabilityMetadata.Severity)) + vulnerabilityRule.CveFixedIns = append(vulnerabilityRule.CveFixedIns, cveFixedInVersion) + vulnerabilityRule.CveDescriptions = append(vulnerabilityRule.CveDescriptions, vulnerabilityMetadata.Description) + vulnerabilityRule.CveCvssScores = append(vulnerabilityRule.CveCvssScores, cvssScore) + vulnerabilityRule.CveOverallScores = append(vulnerabilityRule.CveOverallScores, overallScore) + vulnerabilityRule.CveAttackVectors = append(vulnerabilityRule.CveAttackVectors, attackVector) + vulnerabilityRule.ParsedAttackVectors = append(vulnerabilityRule.ParsedAttackVectors, parsedAttackVector) + + // Common fields + if vulnerabilityMetadata.DataSource != "" { + if !utils.InSlice(vulnerabilityMetadata.DataSource, vulnerabilityRule.CveLinks) { + vulnerabilityRule.CveLinks = append(vulnerabilityRule.CveLinks, vulnerabilityMetadata.DataSource) + } + } + if metasploitURL != "" { + if !utils.InSlice(metasploitURL, vulnerabilityRule.ExploitPOCs) { + vulnerabilityRule.ExploitPOCs = append(vulnerabilityRule.ExploitPOCs, metasploitURL) + } + } + for _, url := range urls { + if !utils.InSlice(url, vulnerabilityRule.URLs) { + vulnerabilityRule.URLs = append(vulnerabilityRule.URLs, url) + } } + + vulnerabilityRules[vulnerability.ID] = vulnerabilityRule } - if len(vulnerabilityRules) > 0 { - _ = saveVulnerabilityRulesInNeo4j(ctx, vulnerabilityRules) + vulnerabilityRulesData := make([]map[string]interface{}, 0) + for _, vulnerabilityRule := range vulnerabilityRules { + vulnerabilityRulesData = append(vulnerabilityRulesData, vulnerabilityRule.ToMap()) + + if len(vulnerabilityRulesData) == neo4jWriteBatchSize { + _ = saveVulnerabilityRulesInNeo4j(ctx, vulnerabilityRulesData) + vulnerabilityRulesData = make([]map[string]interface{}, 0) + } + } + if len(vulnerabilityRulesData) > 0 { + _ = saveVulnerabilityRulesInNeo4j(ctx, vulnerabilityRulesData) } return nil } @@ -554,8 +578,9 @@ func saveVulnerabilityRulesInNeo4j(ctx context.Context, vulnerabilityRules []map if _, err = tx.Run(ctx, ` UNWIND $batch as rule - MERGE (v:VulnerabilityStub{node_id:rule.node_id}) + MERGE (v:VulnerabilityStub:DeepfenceRule{node_id:rule.cve_id}) SET v += rule, + v.type = "vulnerability", v.masked = COALESCE(v.masked, false), v.updated_at = TIMESTAMP()`, map[string]interface{}{"batch": vulnerabilityRules}); err != nil { diff --git a/deepfence_utils/utils/ingesters/vulnerabilities.go b/deepfence_utils/utils/ingesters/vulnerabilities.go index bddb3ac8f9..5138bc52ef 100644 --- a/deepfence_utils/utils/ingesters/vulnerabilities.go +++ b/deepfence_utils/utils/ingesters/vulnerabilities.go @@ -1,9 +1,5 @@ package ingesters -import ( - "fmt" -) - type VulnerabilityScanStatus struct { ScanID string `json:"scan_id"` ScanStatus string `json:"scan_status"` @@ -34,27 +30,22 @@ type Vulnerability struct { } type VulnerabilityRule struct { - CveID string `json:"cve_id"` - CveType string `json:"cve_type"` - CveSeverity string `json:"cve_severity"` - CveFixedIn string `json:"cve_fixed_in"` - CveLink string `json:"cve_link"` - CveDescription string `json:"cve_description"` - CveCvssScore float64 `json:"cve_cvss_score"` - CveOverallScore float64 `json:"cve_overall_score"` - CveAttackVector string `json:"cve_attack_vector"` - URLs []string `json:"urls"` - ExploitPOC string `json:"exploit_poc"` - PackageName string `json:"package_name"` - ParsedAttackVector string `json:"parsed_attack_vector"` - CISAKEV bool `json:"cisa_kev"` - EPSSScore float64 `json:"epss_score"` - Namespace string `json:"namespace"` - NodeID string `json:"node_id"` -} - -func (v *VulnerabilityRule) SetNodeID() { - v.NodeID = fmt.Sprintf("%s-%s", v.CveID, v.Namespace) + CveID string `json:"cve_id"` + CveTypes []string `json:"cve_types"` + CveSeverities []string `json:"cve_severities"` + CveFixedIns []string `json:"cve_fixed_ins"` + CveLinks []string `json:"cve_links"` + CveDescriptions []string `json:"cve_descriptions"` + CveCvssScores []float64 `json:"cve_cvss_scores"` + CveOverallScores []float64 `json:"cve_overall_scores"` + CveAttackVectors []string `json:"cve_attack_vectors"` + URLs []string `json:"urls"` + ExploitPOCs []string `json:"exploit_pocs"` + PackageNames []string `json:"package_names"` + ParsedAttackVectors []string `json:"parsed_attack_vectors"` + CISAKEV bool `json:"cisa_kev"` + EPSSScore float64 `json:"epss_score"` + Namespaces []string `json:"namespaces"` } func (v *VulnerabilityRule) ToMap() map[string]interface{} { @@ -63,23 +54,22 @@ func (v *VulnerabilityRule) ToMap() map[string]interface{} { urls = v.URLs } return map[string]interface{}{ - "cve_id": v.CveID, - "cve_type": v.CveType, - "cve_severity": v.CveSeverity, - "cve_fixed_in": v.CveFixedIn, - "cve_link": v.CveLink, - "cve_description": v.CveDescription, - "cve_cvss_score": v.CveCvssScore, - "cve_overall_score": v.CveOverallScore, - "cve_attack_vector": v.CveAttackVector, - "urls": urls, - "exploit_poc": v.ExploitPOC, - "package_name": v.PackageName, - "parsed_attack_vector": v.ParsedAttackVector, - "cisa_kev": v.CISAKEV, - "epss_score": v.EPSSScore, - "namespace": v.Namespace, - "node_id": v.NodeID, + "cve_id": v.CveID, + "cve_types": v.CveTypes, + "cve_severities": v.CveSeverities, + "cve_fixed_ins": v.CveFixedIns, + "cve_links": v.CveLinks, + "cve_descriptions": v.CveDescriptions, + "cve_cvss_scores": v.CveCvssScores, + "cve_overall_scores": v.CveOverallScores, + "cve_attack_vectors": v.CveAttackVectors, + "urls": urls, + "exploit_pocs": v.ExploitPOCs, + "package_names": v.PackageNames, + "parsed_attack_vectors": v.ParsedAttackVectors, + "cisa_kev": v.CISAKEV, + "epss_score": v.EPSSScore, + "namespaces": v.Namespaces, } } @@ -95,8 +85,8 @@ type VulnerabilityData struct { HasLiveConnection bool `json:"has_live_connection"` } -func (c Vulnerability) Split() (VulnerabilityData, VulnerabilityRule) { - vuln := VulnerabilityData{ +func (c Vulnerability) GetVulnerabilityData() VulnerabilityData { + return VulnerabilityData{ CveID: c.CveID, CveSeverity: c.CveSeverity, CveCausedByPackage: c.CveCausedByPackage, @@ -107,11 +97,4 @@ func (c Vulnerability) Split() (VulnerabilityData, VulnerabilityRule) { InitExploitabilityScore: c.InitExploitabilityScore, HasLiveConnection: c.HasLiveConnection, } - vulnRule := VulnerabilityRule{ - CveID: c.CveID, - Namespace: c.Namespace, - PackageName: c.CveCausedByPackage, - } - vulnRule.SetNodeID() - return vuln, vulnRule } diff --git a/deepfence_worker/go.mod b/deepfence_worker/go.mod index 3295bbacde..9fab8cddeb 100644 --- a/deepfence_worker/go.mod +++ b/deepfence_worker/go.mod @@ -72,6 +72,7 @@ require ( github.com/Microsoft/go-winio v0.6.2 // indirect github.com/Microsoft/hcsshim v0.12.4 // indirect github.com/PagerDuty/go-pagerduty v1.7.0 // indirect + github.com/VirusTotal/gyp v0.9.0 // indirect github.com/XSAM/otelsql v0.31.0 // indirect github.com/acobaugh/osrelease v0.1.0 // indirect github.com/anchore/go-logger v0.0.0-20230725134548-c21dafa1ec5a // indirect diff --git a/deepfence_worker/go.sum b/deepfence_worker/go.sum index 3ae8228a3f..447dd89b7d 100644 --- a/deepfence_worker/go.sum +++ b/deepfence_worker/go.sum @@ -81,6 +81,8 @@ github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PagerDuty/go-pagerduty v1.7.0 h1:S1NcMKECxT5hJwV4VT+QzeSsSiv4oWl1s2821dUqG/8= github.com/PagerDuty/go-pagerduty v1.7.0/go.mod h1:PuFyJKRz1liIAH4h5KVXVD18Obpp1ZXRdxHvmGXooro= +github.com/VirusTotal/gyp v0.9.0 h1:jhOBl93jfStmAcKLa/EcTmdPng5bn5kvJJZqQqJ5R4g= +github.com/VirusTotal/gyp v0.9.0/go.mod h1:nmcW15dQ1657PmMcG9X/EZmp6rTQsyo9g8r6Cz1/AHc= github.com/XSAM/otelsql v0.31.0 h1:AcWI+/BW4ANKyAybZmU9g9kjjSIcDEOFw96ybyM4cDo= github.com/XSAM/otelsql v0.31.0/go.mod h1:iCkLyB/me+QC4yjymXjLimJiX0oklymiKeGxeGDTW24= github.com/acobaugh/osrelease v0.1.0 h1:Yb59HQDGGNhCj4suHaFQQfBps5wyoKLSSX/J/+UifRE= diff --git a/deepfence_worker/ingesters/malware.go b/deepfence_worker/ingesters/malware.go index 2333c86972..051b936058 100644 --- a/deepfence_worker/ingesters/malware.go +++ b/deepfence_worker/ingesters/malware.go @@ -14,7 +14,7 @@ import ( ) func generateMalwareRuleId(metaRule ingestersUtil.MetaRules) string { - return generateHashFromString(metaRule.Author + metaRule.Date + metaRule.RuleName + metaRule.Info + metaRule.Filetype + metaRule.Version) + return fmt.Sprintf("malware-%s", metaRule.RuleName) } func generateHashFromString(s string) string { @@ -50,10 +50,7 @@ func CommitFuncMalware(ctx context.Context, ns string, data []ingestersUtil.Malw if _, err = tx.Run(ctx, ` UNWIND $batch as row WITH row.Rule as rule, row.Malware as malware - MERGE (r:MalwareRule{rule_id:rule.rule_id}) - SET r += rule, - r.masked = COALESCE(r.masked, false), - r.updated_at = TIMESTAMP() + MATCH (r:MalwareRule{rule_id:rule.rule_id}) WITH malware as row, r MERGE (n:Malware{node_id:row.node_id}) SET n+= row, diff --git a/deepfence_worker/ingesters/secrets.go b/deepfence_worker/ingesters/secrets.go index 9b43f0de82..91212729a8 100644 --- a/deepfence_worker/ingesters/secrets.go +++ b/deepfence_worker/ingesters/secrets.go @@ -13,7 +13,7 @@ import ( ) func generateSecretRuleId(r map[string]interface{}) string { - return generateHashFromString(r["name"].(string)) + return fmt.Sprintf("secret-%s", r["name"].(string)) } func CommitFuncSecrets(ctx context.Context, ns string, data []ingestersUtil.Secret) error { @@ -43,13 +43,10 @@ func CommitFuncSecrets(ctx context.Context, ns string, data []ingestersUtil.Secr if _, err = tx.Run(ctx, ` UNWIND $batch as row WITH row.Rule as rule, row.Secret as secret - MERGE (r:SecretRule{rule_id:rule.rule_id}) - SET r+=rule, - r.masked = COALESCE(r.masked, false), - r.updated_at = TIMESTAMP() + MATCH (r:SecretRule{rule_id:rule.rule_id}) WITH secret as row, r MERGE (n:Secret{node_id:row.node_id}) - SET n+= row, + SET n+= row, n.masked = COALESCE(n.masked, r.masked, false), n.updated_at = TIMESTAMP() WITH n, r, row diff --git a/deepfence_worker/ingesters/vulnerabilites.go b/deepfence_worker/ingesters/vulnerabilites.go index 8b68ca1adb..68749aa6ee 100644 --- a/deepfence_worker/ingesters/vulnerabilites.go +++ b/deepfence_worker/ingesters/vulnerabilites.go @@ -40,10 +40,48 @@ func CommitFuncVulnerabilities(ctx context.Context, ns string, data []ingestersU log.Debug().Msgf("Committing %d vulnerabilities", len(dataMap)) + res, err := tx.Run(ctx, ` + UNWIND $batch as row WITH row.data as data + MATCH (v:VulnerabilityStub{node_id:data.cve_id}) + return v.package_names, v.namespaces, v.cve_types, v.cve_attack_vectors, v.cve_fixed_ins`, + map[string]interface{}{"batch": dataMap}) + if err != nil { + log.Error().Msgf(err.Error()) + return err + } + + recs, err := res.Collect(ctx) + if err != nil { + log.Error().Msgf(err.Error()) + return err + } + for i, rec := range recs { + package_names := rec.Values[0].([]any) + namespaces := rec.Values[1].([]any) + cve_types := rec.Values[2].([]any) + cve_attack_vectors := rec.Values[3].([]any) + cve_fixed_ins := rec.Values[4].([]any) + + data := dataMap[i]["data"].(map[string]any) + data["cve_type"] = cve_types[0] + data["cve_attack_vector"] = cve_attack_vectors[0] + data["cve_fixed_in"] = cve_fixed_ins[0] + for j := range package_names { + if data["cve_caused_by_package"].(string) == package_names[j].(string) { + if data["namespace"].(string) == namespaces[j].(string) { + data["cve_type"] = cve_types[j] + data["cve_attack_vector"] = cve_attack_vectors[j] + data["cve_fixed_in"] = cve_fixed_ins[j] + break + } + } + } + + } + if _, err = tx.Run(ctx, ` - UNWIND $batch as row WITH row.rule as rule, row.data as data, - row.scan_id as scan_id, row.node_id as node_id - MATCH (v:VulnerabilityStub{node_id:rule.node_id}) + UNWIND $batch as row WITH row.data as data, row.scan_id as scan_id, row.node_id as node_id + MATCH (v:VulnerabilityStub{node_id:data.cve_id}) MERGE (n:Vulnerability{node_id:node_id}) MERGE (n) -[:IS]-> (v) SET n += data, @@ -64,13 +102,12 @@ func CommitFuncVulnerabilities(ctx context.Context, ns string, data []ingestersU func CVEsToMaps(ms []ingestersUtil.Vulnerability) ([]map[string]interface{}, error) { res := []map[string]interface{}{} for _, v := range ms { - data, rule := v.Split() + data := v.GetVulnerabilityData() res = append(res, map[string]interface{}{ - "rule": utils.ToMap(rule), "data": utils.ToMap(data), "scan_id": v.ScanID, - "node_id": strings.Join([]string{data.CveCausedByPackagePath + data.CveCausedByPackage + rule.CveID}, "_"), + "node_id": strings.Join([]string{data.CveCausedByPackagePath + data.CveCausedByPackage + data.CveID}, "_"), }) } return res, nil diff --git a/deepfence_worker/tasks/reports/data.go b/deepfence_worker/tasks/reports/data.go index 950114d1a2..f537c94b2c 100644 --- a/deepfence_worker/tasks/reports/data.go +++ b/deepfence_worker/tasks/reports/data.go @@ -183,7 +183,7 @@ func getVulnerabilityData(ctx context.Context, params sdkUtils.ReportParams) (*I continue } sort.Slice(result, func(i, j int) bool { - return result[i].CveSeverity < result[j].CveSeverity + return result[i].GetCategory() < result[j].GetCategory() }) nodeWiseData.SeverityCount[s.NodeName] = s.SeverityCounts nodeWiseData.ScanData[s.NodeName] = ScanData[model.Vulnerability]{ @@ -237,13 +237,13 @@ func getMostExploitableVulnData(ctx context.Context, params sdkUtils.ReportParam nodeWiseData.RecordCount += uint64(len(entries)) sevMap := nodeWiseData.SeverityCount[nodeKey] for _, entry := range entries { - count, present := sevMap[entry.CveSeverity] + count, present := sevMap[entry.GetCategory()] if !present { count = 1 } else { count += 1 } - sevMap[entry.CveSeverity] = count + sevMap[entry.GetCategory()] = count } data := Info[model.Vulnerability]{ diff --git a/deepfence_worker/tasks/reports/pdf_vulnerability.go b/deepfence_worker/tasks/reports/pdf_vulnerability.go index e43b2e5012..fe4b2b58ac 100644 --- a/deepfence_worker/tasks/reports/pdf_vulnerability.go +++ b/deepfence_worker/tasks/reports/pdf_vulnerability.go @@ -97,8 +97,8 @@ func vulnerabilityPDF(ctx context.Context, params utils.ReportParams) (core.Docu text.NewCol(3, v.CveCausedByPackage, props.Text{Size: 10, Left: 1, Top: 1, BreakLineStrategy: breakline.DashStrategy}). WithStyle(cellStyle), - text.NewCol(1, v.CveSeverity, - props.Text{Size: 10, Top: 1, Align: align.Center, Style: fontstyle.Bold, Color: colors[v.CveSeverity]}). + text.NewCol(1, v.GetCategory(), + props.Text{Size: 10, Top: 1, Align: align.Center, Style: fontstyle.Bold, Color: colors[v.GetCategory()]}). WithStyle(cellStyle), text.NewCol(4, truncateText(v.CveDescription, 80), props.Text{Size: 10, Left: 1, Top: 1, BreakLineStrategy: breakline.EmptySpaceStrategy}). diff --git a/docs/static/img/threatmapper-intro.mp4 b/docs/static/img/threatmapper-intro.mp4 deleted file mode 100644 index 13fe787c91..0000000000 Binary files a/docs/static/img/threatmapper-intro.mp4 and /dev/null differ diff --git a/golang_deepfence_sdk b/golang_deepfence_sdk index 4943c14781..226fd1e1e6 160000 --- a/golang_deepfence_sdk +++ b/golang_deepfence_sdk @@ -1 +1 @@ -Subproject commit 4943c14781c54befc03e4011650a369de6926137 +Subproject commit 226fd1e1e6bb7b9f4e179d6f738bb7cd9c55dd72