From 2696ebcf524579be98f46bba740a9e0c4a4751e4 Mon Sep 17 00:00:00 2001 From: Aleksejs Date: Wed, 26 Jun 2024 15:37:21 +0300 Subject: [PATCH] MISP plugin added --- .gitignore | 6 +- Dockerfile.example | 3 +- Makefile.example | 33 ++- README.md | 3 +- VERSION | 2 +- assets/js/graph.js | 7 +- assets/js/sql-autocomplete.js | 6 +- go.mod | 20 +- go.sum | 43 +-- pdk/stats.go | 4 +- plugins/src/misp/README.md | 104 +++++++ plugins/src/misp/convert.go | 30 ++ plugins/src/misp/misp.go | 462 +++++++++++++++++++++++++++++ plugins/src/misp/misp_test.go | 78 +++++ plugins/src/misp/package.go | 528 ++++++++++++++++++++++++++++++++++ plugins/src/misp/plugin.go | 33 +++ plugins/src/misp/select.go | 218 ++++++++++++++ 17 files changed, 1522 insertions(+), 58 deletions(-) create mode 100644 plugins/src/misp/README.md create mode 100644 plugins/src/misp/convert.go create mode 100644 plugins/src/misp/misp.go create mode 100644 plugins/src/misp/misp_test.go create mode 100644 plugins/src/misp/package.go create mode 100644 plugins/src/misp/plugin.go create mode 100644 plugins/src/misp/select.go diff --git a/.gitignore b/.gitignore index a3fedd7..135b1d7 100755 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,5 @@ -certs/graph.* -certs/graph-* -certs/certca2.crt -certs/cluster.crt +certs/* +!certs/graphoscope.* definitions/*.yaml !definitions/sources/*.example !definitions/processors/*.example diff --git a/Dockerfile.example b/Dockerfile.example index 491f299..273837e 100644 --- a/Dockerfile.example +++ b/Dockerfile.example @@ -29,11 +29,12 @@ RUN go build -buildmode=plugin -ldflags="-w" -o /go/plugins/sources/elasticsearc RUN go build -buildmode=plugin -ldflags="-w" -o /go/plugins/sources/http.so plugins/src/http/*.go RUN go build -buildmode=plugin -ldflags="-w" -o /go/plugins/sources/rest.so plugins/src/rest/*.go RUN go build -buildmode=plugin -ldflags="-w" -o /go/plugins/sources/mongodb.so plugins/src/mongodb/*.go -RUN go build -buildmode=plugin -ldflags="-w" -o /go/plugins/sources/pastelyzer.so plugins/src/pastelyzer/*.go RUN go build -buildmode=plugin -ldflags="-w" -o /go/plugins/sources/postgresql.so plugins/src/postgresql/*.go RUN go build -buildmode=plugin -ldflags="-w" -o /go/plugins/sources/redis.so plugins/src/redis/*.go RUN go build -buildmode=plugin -ldflags="-w" -o /go/plugins/sources/mysql.so plugins/src/mysql/*.go RUN go build -buildmode=plugin -ldflags="-w" -o /go/plugins/sources/file-csv.so plugins/src/file/csv/*.go +RUN go build -buildmode=plugin -ldflags="-w" -o /go/plugins/sources/misp.so plugins/src/misp/*.go +RUN go build -buildmode=plugin -ldflags="-w" -o /go/plugins/sources/pastelyzer.so plugins/src/pastelyzer/*.go RUN go build -buildmode=plugin -ldflags="-w" -o /go/plugins/sources/abuseipdb.so plugins/src/abuseipdb/*.go RUN go build -buildmode=plugin -ldflags="-w" -o /go/plugins/sources/hashlookup.so plugins/src/hashlookup/*.go RUN go build -buildmode=plugin -ldflags="-w" -o /go/plugins/sources/circl_passive_ssl.so plugins/src/circl_passive_ssl/*.go diff --git a/Makefile.example b/Makefile.example index 322785a..c94a4e6 100644 --- a/Makefile.example +++ b/Makefile.example @@ -170,22 +170,23 @@ uninstall-remote: # Build plugins locally, mainly for development plugins-local: - go build -buildmode=plugin -ldflags="-w" -o plugins/sources/elasticsearch.v7.so plugins/src/elasticsearch.v7/*.go - go build -buildmode=plugin -ldflags="-w" -o plugins/sources/http.so plugins/src/http/*.go - go build -buildmode=plugin -ldflags="-w" -o plugins/sources/rest.so plugins/src/rest/*.go - go build -buildmode=plugin -ldflags="-w" -o plugins/sources/mongodb.so plugins/src/mongodb/*.go - go build -buildmode=plugin -ldflags="-w" -o plugins/sources/pastelyzer.so plugins/src/pastelyzer/*.go - go build -buildmode=plugin -ldflags="-w" -o plugins/sources/postgresql.so plugins/src/postgresql/*.go - go build -buildmode=plugin -ldflags="-w" -o plugins/sources/redis.so plugins/src/redis/*.go - go build -buildmode=plugin -ldflags="-w" -o plugins/sources/mysql.so plugins/src/mysql/*.go - go build -buildmode=plugin -ldflags="-w" -o plugins/sources/file-csv.so plugins/src/file/csv/*.go - go build -buildmode=plugin -ldflags="-w" -o plugins/sources/abuseipdb.so plugins/src/abuseipdb/*.go - go build -buildmode=plugin -ldflags="-w" -o plugins/sources/hashlookup.so plugins/src/hashlookup/*.go + go build -buildmode=plugin -ldflags="-w" -o plugins/sources/elasticsearch.v7.so plugins/src/elasticsearch.v7/*.go + go build -buildmode=plugin -ldflags="-w" -o plugins/sources/http.so plugins/src/http/*.go + go build -buildmode=plugin -ldflags="-w" -o plugins/sources/rest.so plugins/src/rest/*.go + go build -buildmode=plugin -ldflags="-w" -o plugins/sources/mongodb.so plugins/src/mongodb/*.go + go build -buildmode=plugin -ldflags="-w" -o plugins/sources/postgresql.so plugins/src/postgresql/*.go + go build -buildmode=plugin -ldflags="-w" -o plugins/sources/redis.so plugins/src/redis/*.go + go build -buildmode=plugin -ldflags="-w" -o plugins/sources/mysql.so plugins/src/mysql/*.go + go build -buildmode=plugin -ldflags="-w" -o plugins/sources/file-csv.so plugins/src/file/csv/*.go + go build -buildmode=plugin -ldflags="-w" -o plugins/sources/misp.so plugins/src/misp/*.go + go build -buildmode=plugin -ldflags="-w" -o plugins/sources/pastelyzer.so plugins/src/pastelyzer/*.go + go build -buildmode=plugin -ldflags="-w" -o plugins/sources/abuseipdb.so plugins/src/abuseipdb/*.go + go build -buildmode=plugin -ldflags="-w" -o plugins/sources/hashlookup.so plugins/src/hashlookup/*.go go build -buildmode=plugin -ldflags="-w" -o plugins/sources/circl_passive_ssl.so plugins/src/circl_passive_ssl/*.go CGO_CFLAGS="-g -O2 -Wno-return-local-addr" go build -buildmode=plugin -ldflags="-w" -o plugins/sources/sqlite.so plugins/src/sqlite/*.go - go build -buildmode=plugin -ldflags="-w" -o plugins/processors/taxonomy.so plugins/src/taxonomy/*.go - go build -buildmode=plugin -ldflags="-w" -o plugins/processors/modify.so plugins/src/modify/*.go + go build -buildmode=plugin -ldflags="-w" -o plugins/processors/taxonomy.so plugins/src/taxonomy/*.go + go build -buildmode=plugin -ldflags="-w" -o plugins/processors/modify.so plugins/src/modify/*.go go build -buildmode=plugin -ldflags="-w" -o /dev/null plugins/src/template/*.go @@ -195,21 +196,23 @@ test: go test plugins/src/http/*.go go test plugins/src/rest/*.go go test plugins/src/mongodb/*.go - go test plugins/src/pastelyzer/*.go go test plugins/src/postgresql/*.go go test plugins/src/redis/*.go go test plugins/src/mysql/*.go go test plugins/src/file/csv/*.go + go test plugins/src/misp/*.go + go test plugins/src/pastelyzer/*.go go test plugins/src/abuseipdb/*.go go test plugins/src/hashlookup/*.go go test plugins/src/circl_passive_ssl/*.go CGO_CFLAGS="-g -O2 -Wno-return-local-addr" go test plugins/src/sqlite/*.go go test plugins/src/taxonomy/*.go + go test plugins/src/modify/*.go # Check for Golang errors & inefficient code. Install with: # curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh # mv bin/golangci-lint "$GOPATH/bin/" && rm -rf bin lint: - golangci-lint run --enable=golint --enable=gosec --enable=maligned --enable=prealloc --skip-dirs "(ideas)" ./... + golangci-lint run --timeout=2m --enable=revive --enable=gosec --enable=govet --enable=prealloc --exclude-dirs "(ideas)" ./... # golint . diff --git a/README.md b/README.md index 54713ea..569538b 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,7 @@ Available plugins are in [plugins/src](plugins/src): - AbuseIPDB - Hashlookup - CIRCL Passive SSL +- MISP 3rd party compiled `*.so` plugins should be placed in [plugins/sources](plugins/sources) directory. @@ -199,6 +200,7 @@ Response example for the first query: - [ ] Implement other SQL features, like `NOT BETWEEN` - [ ] Filters `Edit` button doesn't work if data source is not available any more - [ ] API can return an image instead of JSON +- [ ] Use the official package for the Elasticsearch plugin - [ ] Data source plugins: - [ ] RTIR - [ ] MS SQL @@ -206,7 +208,6 @@ Response example for the first query: - [ ] Apache Cassandra - [ ] Genji - [ ] Presto - - [ ] MISP - [ ] VirusTotal - [ ] Shodan - [ ] General TCP diff --git a/VERSION b/VERSION index 6de6d05..a5c51e1 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.5.7 \ No newline at end of file +2.5.8 \ No newline at end of file diff --git a/assets/js/graph.js b/assets/js/graph.js index 479af17..af1f653 100644 --- a/assets/js/graph.js +++ b/assets/js/graph.js @@ -572,8 +572,13 @@ class Graph { for (var i = 0; i < selected.length; i++) { const node = this.application.graph.network.body.nodes[selected[i]]; - this.application.search.query('FROM ' + source + ' WHERE ' + node.options.search + '=\'' + node.options.attributes[node.options.group] + '\''); + if (node.options.search === '') { + this.application.modal.error('Can not expand graph!', 'Search for this node type is not supported yet!'); + return; + } + + this.application.search.query('FROM ' + source + ' WHERE ' + node.options.search + '=\'' + node.options.attributes[node.options.group] + '\''); console.log('Expanding by', node.options.search, '=', node.id, 'from', source); } } diff --git a/assets/js/sql-autocomplete.js b/assets/js/sql-autocomplete.js index 75a8c91..b35a2f1 100644 --- a/assets/js/sql-autocomplete.js +++ b/assets/js/sql-autocomplete.js @@ -78,15 +78,15 @@ class SQLAutocomplete { option.innerHTML = '' + field.substr(0, word.length) + ''; option.innerHTML += field.substr(word.length); - // Insert a data attribute that will hold the current array item's value + // Insert data attribute that will hold current array item's value option.dataset.field = field; // Execute a function when someone clicks on the item value (DIV element) option.addEventListener('click', (e) => { let value = option.dataset.field; - // If field contains "-" character - backticks mush be added - if (value.includes('-')) + // If field name contains special characters - backticks must be added + if (value.includes('-') || value.includes('|') || value.includes('/')) value = '`' + value + '`'; // Insert the value for the autocomplete text field diff --git a/go.mod b/go.mod index 9b846f0..c3cceba 100644 --- a/go.mod +++ b/go.mod @@ -5,25 +5,26 @@ go 1.22 toolchain go1.22.2 require ( + github.com/0xrawsec/golang-utils v1.1.8 github.com/Jeffail/gabs/v2 v2.7.0 github.com/blastrain/vitess-sqlparser v0.0.0-20201030050434-a139afbb1aba github.com/georgysavva/scany v1.2.1 github.com/go-sql-driver/mysql v1.7.1 github.com/google/uuid v1.6.0 github.com/gorilla/securecookie v1.1.2 - github.com/gorilla/sessions v1.2.2 - github.com/gorilla/websocket v1.5.1 + github.com/gorilla/sessions v1.3.0 + github.com/gorilla/websocket v1.5.3 github.com/jackc/pgx/v4 v4.18.1 github.com/mattn/go-sqlite3 v1.14.20 github.com/mithrandie/csvq-driver v1.7.0 github.com/olekukonko/tablewriter v0.0.5 github.com/olivere/elastic/v7 v7.0.32 github.com/redis/go-redis/v9 v9.4.0 - github.com/rs/zerolog v1.32.0 + github.com/rs/zerolog v1.33.0 github.com/umpc/go-sortedmap v0.0.0-20180422175548-64ab94c482f4 github.com/yukithm/json2csv v0.1.2 - go.mongodb.org/mongo-driver v1.15.0 - golang.org/x/crypto v0.22.0 + go.mongodb.org/mongo-driver v1.15.1 + golang.org/x/crypto v0.24.0 golang.org/x/sync v0.7.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v3 v3.0.1 @@ -43,7 +44,7 @@ require ( github.com/jackc/puddle v1.3.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/juju/errors v1.0.0 // indirect - github.com/klauspost/compress v1.17.8 // indirect + github.com/klauspost/compress v1.17.9 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect @@ -60,8 +61,7 @@ require ( github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20240424034433-3c2c7870ae76 // indirect - golang.org/x/net v0.24.0 // indirect - golang.org/x/sys v0.19.0 // indirect - golang.org/x/term v0.19.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/term v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect ) diff --git a/go.sum b/go.sum index b0f4c7d..a3060b0 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/0xrawsec/golang-utils v1.1.8 h1:9TzAKzC7+V2IXaV/3Y1aXiUFHeShL3BXcfL6BheAEH0= +github.com/0xrawsec/golang-utils v1.1.8/go.mod h1:DADTtCFY10qXjWmUVhhJqQIZdSweaHH4soYUDEi8mj0= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Jeffail/gabs/v2 v2.7.0 h1:Y2edYaTcE8ZpRsR2AtmPu5xQdFDIthFG0jYhu5PY8kg= github.com/Jeffail/gabs/v2 v2.7.0/go.mod h1:dp5ocw1FvBBQYssgHsG7I1WYsiLRtkUaB1FEtSwvNUw= @@ -50,10 +52,10 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/securecookie v1.1.2 h1:YCIWL56dvtr73r6715mJs5ZvhtnY73hBvEF8kXD8ePA= github.com/gorilla/securecookie v1.1.2/go.mod h1:NfCASbcHqRSY+3a8tlWJwsQap2VX5pwzwo4h3eOamfo= -github.com/gorilla/sessions v1.2.2 h1:lqzMYz6bOfvn2WriPUjNByzeXIlVzURcPmgMczkmTjY= -github.com/gorilla/sessions v1.2.2/go.mod h1:ePLdVu+jbEgHH+KWw8I1z2wqd0BAdAQh/8LRvBeoNcQ= -github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= -github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= +github.com/gorilla/sessions v1.3.0 h1:XYlkq7KcpOB2ZhHBPv5WpjMIxrQosiZanfoy1HLZFzg= +github.com/gorilla/sessions v1.3.0/go.mod h1:ePLdVu+jbEgHH+KWw8I1z2wqd0BAdAQh/8LRvBeoNcQ= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/hashicorp/go-version v1.0.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= @@ -130,10 +132,11 @@ github.com/juju/errors v1.0.0/go.mod h1:B5x9thDqx0wIMH3+aLIMP9HjItInYWObRovoCFM5 github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= github.com/juju/testing v0.0.0-20191001232224-ce9dec17d28b/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= -github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= @@ -195,6 +198,7 @@ github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCz github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.0/go.mod h1:NxmoDg/QLVWluQDUYG7XBZTLUpKeFa8e3aMf1BfjyHk= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/redis/go-redis/v9 v9.4.0 h1:Yzoz33UZw9I/mFhx4MNrB6Fk+XHO1VukNcCa1+lwyKk= @@ -207,8 +211,8 @@ github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= -github.com/rs/zerolog v1.32.0 h1:keLypqrlIjaFsbmJOBdB/qvyF8KEtCWHwobLp5l/mQ0= -github.com/rs/zerolog v1.32.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= +github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= +github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= @@ -246,8 +250,8 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t github.com/yukithm/json2csv v0.1.2 h1:b2aIY9+TOY5Wss9lCku4wjqnQrENv5Ix1G0ZHN1FE2Q= github.com/yukithm/json2csv v0.1.2/go.mod h1:Ul6ZenFV94YeUm08AqppOd+/hB9JsmiU4KXPs9ZvgwQ= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= -go.mongodb.org/mongo-driver v1.15.0 h1:rJCKC8eEliewXjZGf0ddURtl7tTVy1TK3bfl0gkUSLc= -go.mongodb.org/mongo-driver v1.15.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= +go.mongodb.org/mongo-driver v1.15.1 h1:l+RvoUOoMXFmADTLfYDm7On9dRm7p4T80/lEQM+r7HU= +go.mongodb.org/mongo-driver v1.15.1/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -273,8 +277,8 @@ golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= -golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= @@ -286,8 +290,6 @@ golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= -golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= @@ -312,14 +314,14 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= -golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= -golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= +golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= +golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= golang.org/x/text v0.0.0-20180302201248-b7ef84aaf62a/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -330,10 +332,11 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190320215829-36c10c0a621f/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= diff --git a/pdk/stats.go b/pdk/stats.go index 748a56b..53e97d0 100644 --- a/pdk/stats.go +++ b/pdk/stats.go @@ -85,9 +85,9 @@ func (s *Stats) ToJSON(source string) (map[string]interface{}, error) { // We want Top 10 here and started from i == 1 if i > 9 { break - } else { - i++ } + + i++ } json[k] = group diff --git a/plugins/src/misp/README.md b/plugins/src/misp/README.md new file mode 100644 index 0000000..de29802 --- /dev/null +++ b/plugins/src/misp/README.md @@ -0,0 +1,104 @@ +# MISP plugin + +Plugin allows to query a MISP instance. +Can search for attributes by [type/]value and for events by ID/UUID, with or without datetime range. + +To search for event: +```sql +FROM misp WHERE event='000000' +FROM misp WHERE event='00000000-0000-0000-80f3-8e92723639a8' +``` +To search for attribute of any type: +```sql +FROM misp WHERE attribute='8.8.8.8' +``` +To search for attribute of specific type and datetime range: +```sql +FROM misp WHERE hostname='example.com' and datetime BETWEEN '2024-05-04T11:30:14.000Z' AND '2024-06-04T11:30:14.000Z' +``` + +More info at: +https://www.misp-project.org/openapi/#tag/Attributes/operation/restSearchAttributes +https://www.misp-project.org/openapi/#tag/Events/operation/restSearchEvents + +`curl` to test: +```sh +curl 'https://localhost:443/api?uuid=auth-key&sql=FROM+misp+WHERE+attribute=%278.8.8.8%27' +``` + +Compile with: +```sh +go build -buildmode=plugin -ldflags="-w" -o misp.so ./*.go +``` + +# Access details + +Source YAML definition's `access` fields: +- **protocol**: "https" or "http" +- **host**: instance's hostname +- **apiKey**: user's unique API access key +- **caCertPath**: CA file path +- **certPath**: certificate file path +- **keyPath**: key file path + + +# YAML example + +As MISP has a very large amount of different attribute types, graph relations are generated on the fly, no need to put them all in a YAML config. So it is enough to start with: +``` +name: misp +label: MISP +icon: share square + +plugin: misp +inGlobal: true +includeDatetime: true +supportsSQL: false + +access: + protocol: https + host: misp.example.com + apiKey: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + caCertPath: certs/ca.crt + certPath: certs/misp.crt + keyPath: certs/misp.key + +queryFields: ["event", "attribute", "md5", "sha1", "sha256", "filename", "pdb", "filename|md5", "filename|sha1", "filename|sha256", "ip-src", + "ip-dst", "hostname", "domain", "domain|ip", "email", "email-src", "eppn", "email-dst", "email-subject", "email-attachment", + "email-body", "float", "git-commit-id", "url", "http-method", "user-agent", "ja3-fingerprint-md5", "jarm-fingerprint", "favicon-mmh3", + "hassh-md5", "hasshserver-md5", "regkey", "regkey|value", "AS", "snort", "bro", "zeek", "community-id", "pattern-in-file", + "pattern-in-traffic", "pattern-in-memory", "pattern-filename", "pgp-public-key", "pgp-private-key", "yara", "stix2-pattern", "sigma", + "gene", "kusto-query", "mime-type", "identity-card-number", "cookie", "vulnerability", "cpe", "weakness", "attachment", + "malware-sample", "link", "comment", "text", "hex", "other", "named pipe", "mutex", "process-state", "target-user", "target-email", + "target-machine", "target-org", "target-location", "target-external", "btc", "dash", "xmr", "iban", "bic", "bank-account-nr", + "aba-rtn", "bin", "cc-number", "prtn", "phone-number", "threat-actor", "campaign-name", "campaign-id", "malware-type", "uri", + "authentihash", "vhash", "ssdeep", "imphash", "telfhash", "pehash", "impfuzzy", "sha224", "sha384", "sha512", "sha512/224", + "sha512/256", "sha3-224", "sha3-256", "sha3-384", "sha3-512", "tlsh", "cdhash", "filename|authentihash", "filename|vhash", + "filename|ssdeep", "filename|imphash", "filename|impfuzzy", "filename|pehash", "filename|sha224", "filename|sha384", + "filename|sha512", "filename|sha512/224", "filename|sha512/256", "filename|sha3-224", "filename|sha3-256", "filename|sha3-384", + "filename|sha3-512", "filename|tlsh", "windows-scheduled-task", "windows-service-name", "windows-service-displayname", + "whois-registrant-email", "whois-registrant-phone", "whois-registrant-name", "whois-registrant-org", "whois-registrar", + "whois-creation-date", "x509-fingerprint-sha1", "x509-fingerprint-md5", "x509-fingerprint-sha256", "dns-soa-email", "size-in-bytes", + "counter", "datetime", "port", "ip-dst|port", "ip-src|port", "hostname|port", "mac-address", "mac-eui-64", "email-dst-display-name", + "email-src-display-name", "email-header", "email-reply-to", "email-x-mailer", "email-mime-boundary", "email-thread-index", + "email-message-id", "github-username", "github-repository", "github-organisation", "jabber-id", "twitter-id", "dkim", + "dkim-signature", "first-name", "middle-name", "last-name", "full-name", "date-of-birth", "place-of-birth", "gender", + "passport-number", "passport-country", "passport-expiration", "redress-number", "nationality", "visa-number", + "issue-date-of-the-visa", "primary-residence", "country-of-residence", "special-service-request", "frequent-flyer-number", + "travel-details", "payment-details", "place-port-of-original-embarkation", "place-port-of-clearance", + "place-port-of-onward-foreign-destination", "passenger-name-record-locator-number", "mobile-application-id", "chrome-extension-id", + "cortex", "boolean", "anonymised"] + +statsFields: + - Category + - Org + - Orgc + - Published + - Distribution + - ToIDS + +replaceFields: + domain: attribute + ip: attribute + email: attribute +``` diff --git a/plugins/src/misp/convert.go b/plugins/src/misp/convert.go new file mode 100644 index 0000000..a28195f --- /dev/null +++ b/plugins/src/misp/convert.go @@ -0,0 +1,30 @@ +/* + * SQL to the field/value list convertor + */ + +package main + +import ( + "github.com/blastrain/vitess-sqlparser/sqlparser" +) + +/* + * Convert SQL query to the [field, value] or + * [field, value, datetime from, datetime to] if datetime exists + */ +func (p *plugin) convert(sel *sqlparser.Select) ([]string, error) { + + // Handle WHERE. + // Top level node pass in an empty interface + // to tell the children this is root. + // Is there any better way? + var rootParent sqlparser.Expr + + // List of requested fields & values + fields, err := handleSelectWhere(&sel.Where.Expr, true, &rootParent) + if err != nil { + return nil, err + } + + return fields, nil +} diff --git a/plugins/src/misp/misp.go b/plugins/src/misp/misp.go new file mode 100644 index 0000000..da29e8f --- /dev/null +++ b/plugins/src/misp/misp.go @@ -0,0 +1,462 @@ +package main + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "net/http" + "os" + "regexp" + "strconv" + "strings" + "sync" + "time" + + "github.com/blastrain/vitess-sqlparser/sqlparser" + "github.com/cert-lv/graphoscope/pdk" + "github.com/umpc/go-sortedmap" + "github.com/umpc/go-sortedmap/desc" +) + +/* + * Check "pdk/plugin.go" for the built-in plugin functions description + */ + +func (p *plugin) Conf() *pdk.Source { + return p.source +} + +func (p *plugin) Setup(source *pdk.Source, limit int) error { + + // Validate necessary parameters + if source.Access["protocol"] != "http" && source.Access["protocol"] != "https" { + return fmt.Errorf("'access.protocol' must be 'http[s]'") + } + + if source.Access["host"] == "" { + return fmt.Errorf("'access.host' is not defined") + } + + if source.Access["apiKey"] == "" { + return fmt.Errorf("'access.apiKey' is not defined") + } + + // Store settings + p.source = source + p.limit = limit + p.protocol = source.Access["protocol"] + p.host = source.Access["host"] + p.apiKey = source.Access["apiKey"] + p.types = make(map[string]bool) + + p.caCertPath = source.Access["caCertPath"] + p.certPath = source.Access["certPath"] + p.keyPath = source.Access["keyPath"] + + // Set possible variable type & searching fields + for _, relation := range source.Relations { + for _, types := range relation.From.VarTypes { + types.RegexCompiled = regexp.MustCompile(types.Regex) + } + + for _, types := range relation.To.VarTypes { + types.RegexCompiled = regexp.MustCompile(types.Regex) + } + } + + // fmt.Printf("MISP %s: %#v\n\n", source.Name, p) + return nil +} + +func (p *plugin) Fields() ([]string, error) { + return p.source.QueryFields, nil +} + +func (p *plugin) Search(stmt *sqlparser.Select) ([]map[string]interface{}, map[string]interface{}, map[string]interface{}, error) { + + // Storage for the results to return + results := []map[string]interface{}{} + + // Convert SQL statement + searchField, err := p.convert(stmt) + if err != nil { + return nil, nil, nil, err + } + + /* + * Send indicators to get results back + */ + entries, debug, err := p.request(searchField) + if err != nil { + return nil, nil, debug, err + } + + //fmt.Printf("MISP response:\n%v\n", body) + + // Struct to store statistics data + // when the amount of returned entries is too large + stats := pdk.NewStats() + + for _, field := range p.source.StatsFields { + stats.Fields[field] = sortedmap.New(10, desc.Int) + } + + /* + * Receive hits and deserialize them + */ + + mx := &sync.Mutex{} + unique := make(map[string]bool) + counter := 0 + + // Process results + for _, entry := range entries { + + // Stop when results count is too big + if counter >= p.limit { + top, err := stats.ToJSON(p.source.Name) + if err != nil { + return nil, nil, debug, err + } + + return nil, top, debug, nil + } + + // Update stats + for _, field := range p.source.StatsFields { + stats.Update(entry, field) + } + + pdk.CreateRelations(p.source, entry, unique, &counter, mx, &results) + } + + return results, nil, debug, nil +} + +// request connects to the MISP instance and returns the response +func (p *plugin) request(searchField []string) ([]map[string]interface{}, map[string]interface{}, error) { + + // Debug info + debug := make(map[string]interface{}) + var entries []map[string]interface{} + + // Load TLS certificates + clientTLSCert, err := tls.LoadX509KeyPair(p.certPath, p.keyPath) + if err != nil { + return nil, debug, fmt.Errorf("Error loading certificate and key file: %s", err.Error()) + } + + // Configure the client to trust TLS server certs issued by a CA + certPool, err := x509.SystemCertPool() + if err != nil { + return nil, debug, fmt.Errorf("Can't create SystemCertPool: %s", err.Error()) + } + + if caCertPEM, err := os.ReadFile(p.caCertPath); err != nil { + return nil, debug, fmt.Errorf("Can't read cert CA file: %s", err.Error()) + } else if ok := certPool.AppendCertsFromPEM(caCertPEM); !ok { + return nil, debug, fmt.Errorf("Invalid cert CA file") + } + + tlsConfig := &tls.Config{ + RootCAs: certPool, + Certificates: []tls.Certificate{clientTLSCert}, + MinVersion: tls.VersionTLS12, // At least TLS v1.2 is recommended + } + tr := &http.Transport{ + TLSClientConfig: tlsConfig, + } + client := &http.Client{Transport: tr} + + con := MispCon{p.protocol, p.host, p.apiKey, client} + var mq MispQuery + + // Search for event by ID/UUID + if searchField[0] == "event" { + if len(searchField) == 2 { + mq = MispEventQuery{ + EventID: searchField[1], + } + } else { + mq = MispEventQuery{ + EventID: searchField[1], + From: searchField[2], + To: searchField[3], + } + } + + // Search for attribute of any type + } else if searchField[0] == "attribute" { + if len(searchField) == 2 { + mq = MispAttributeQuery{ + Value: searchField[1], + } + } else { + mq = MispAttributeQuery{ + Value: searchField[1], + From: searchField[2], + To: searchField[3], + } + } + + // Search for attribute of specific type + } else { + if len(searchField) == 2 { + mq = MispAttributeQuery{ + Type: searchField[0], + Value: searchField[1], + } + } else { + mq = MispAttributeQuery{ + Type: searchField[0], + Value: searchField[1], + From: searchField[2], + To: searchField[3], + } + } + } + + debug["query"] = fmt.Sprint(mq) + + mr, err := con.Search(mq) + if err != nil { + return nil, debug, fmt.Errorf("Failed to search: %s", err.Error()) + } + + if searchField[0] == "event" { + for mri := range mr.Iter() { + for _, a := range mri.(MispEvent).Attribute { + entry := map[string]interface{}{ + "EventID": mri.(MispEvent).ID, + "EventLabel": mri.(MispEvent).ID + ": " + mri.(MispEvent).Info, + "OrgcID": mri.(MispEvent).OrgcID, + "OrgID": mri.(MispEvent).OrgID, + "Date": mri.(MispEvent).Date, + "ThreatLevelID": mri.(MispEvent).ThreatLevelID, + "Info": mri.(MispEvent).Info, + "Published": mri.(MispEvent).Published, + "EventUUID": mri.(MispEvent).UUID, + "AttributeCount": mri.(MispEvent).AttributeCount, + "Analysis": mri.(MispEvent).Analysis, + "EventDistribution": mri.(MispEvent).Distribution, + "ProposalEmailLock": mri.(MispEvent).ProposalEmailLock, + "Locked": mri.(MispEvent).Locked, + "EventSharingGroupID": mri.(MispEvent).SharingGroupID, + "Org": mri.(MispEvent).Org.Name, + "Orgc": mri.(MispEvent).Orgc.Name, + + a.Type: a.Value, + + "ID": a.ID, + "UUID": a.UUID, + "SharingGroupID": a.SharingGroupID, + "Distribution": a.Distribution, + "Category": a.Category, + "ToIDS": a.ToIDS, + "Deleted": a.Deleted, + "Comment": a.Comment, + } + + // Get tags + eventTags := []string{} + for _, tag := range mri.(MispEvent).Tag { + eventTags = append(eventTags, tag.Name) + } + + attributeTags := []string{} + for _, tag := range a.Tag { + attributeTags = append(attributeTags, tag.Name) + } + + entry["EventTags"] = strings.Join(eventTags[:], ", ") + entry["AttributeTags"] = strings.Join(attributeTags[:], ", ") + + // Convert timestamps to datetime + eventStrTimestamp, err := strconv.ParseInt(mri.(MispEvent).StrTimestamp, 10, 64) + if err != nil { + return nil, debug, fmt.Errorf("Failed to parse event's StrTimestamp: %s", mri.(MispEvent).StrTimestamp) + } + + strPublishedTimestamp, err := strconv.ParseInt(mri.(MispEvent).StrPublishedTimestamp, 10, 64) + if err != nil { + return nil, debug, fmt.Errorf("Failed to parse event's StrPublishedTimestamp: %s", mri.(MispEvent).StrPublishedTimestamp) + } + + strTimestamp, err := strconv.ParseInt(a.StrTimestamp, 10, 64) + if err != nil { + return nil, debug, fmt.Errorf("Failed to parse attribute's StrTimestamp: %s", a.StrTimestamp) + } + + entry["EventStrTimestamp"] = time.Unix(eventStrTimestamp, 0) + entry["StrPublishedTimestamp"] = time.Unix(strPublishedTimestamp, 0) + entry["StrTimestamp"] = time.Unix(strTimestamp, 0) + + entries = append(entries, entry) + p.generateRelations(a.Type) + } + + for _, o := range mri.(MispEvent).Object { + entry := map[string]interface{}{ + "EventID": mri.(MispEvent).ID, + "EventLabel": mri.(MispEvent).ID + ": " + mri.(MispEvent).Info, + "OrgcID": mri.(MispEvent).OrgcID, + "OrgID": mri.(MispEvent).OrgID, + "Date": mri.(MispEvent).Date, + "ThreatLevelID": mri.(MispEvent).ThreatLevelID, + "Info": mri.(MispEvent).Info, + "Published": mri.(MispEvent).Published, + "EventUUID": mri.(MispEvent).UUID, + "AttributeCount": mri.(MispEvent).AttributeCount, + "Analysis": mri.(MispEvent).Analysis, + "EventDistribution": mri.(MispEvent).Distribution, + "ProposalEmailLock": mri.(MispEvent).ProposalEmailLock, + "Locked": mri.(MispEvent).Locked, + "EventSharingGroupID": mri.(MispEvent).SharingGroupID, + "Org": mri.(MispEvent).Org.Name, + "Orgc": mri.(MispEvent).Orgc.Name, + + "Label": o.Name + ": " + o.ID, + "ID": o.ID, + "Name": o.Name, + "Description": o.Description, + "UUID": o.UUID, + "SharingGroupID": o.SharingGroupID, + "Distribution": o.Distribution, + "Deleted": o.Deleted, + "Comment": o.Comment, + "FirstSeen": o.FirstSeen, + "LastSeen": o.LastSeen, + } + + // Get tags + eventTags := []string{} + for _, tag := range mri.(MispEvent).Tag { + eventTags = append(eventTags, tag.Name) + } + + entry["EventTags"] = strings.Join(eventTags[:], ", ") + + // Convert timestamps to datetime + eventStrTimestamp, err := strconv.ParseInt(mri.(MispEvent).StrTimestamp, 10, 64) + if err != nil { + return nil, debug, fmt.Errorf("Failed to parse event's StrTimestamp: %s", mri.(MispEvent).StrTimestamp) + } + + strPublishedTimestamp, err := strconv.ParseInt(mri.(MispEvent).StrPublishedTimestamp, 10, 64) + if err != nil { + return nil, debug, fmt.Errorf("Failed to parse event's StrPublishedTimestamp: %s", mri.(MispEvent).StrPublishedTimestamp) + } + + strTimestamp, err := strconv.ParseInt(o.StrTimestamp, 10, 64) + if err != nil { + return nil, debug, fmt.Errorf("Failed to parse objects's StrTimestamp: %s", o.StrTimestamp) + } + + entry["EventStrTimestamp"] = time.Unix(eventStrTimestamp, 0) + entry["StrPublishedTimestamp"] = time.Unix(strPublishedTimestamp, 0) + entry["StrTimestamp"] = time.Unix(strTimestamp, 0) + + entries = append(entries, entry) + p.generateRelations("Label") + } + } + } else { + for a := range mr.Iter() { + entry := map[string]interface{}{ + a.(MispAttribute).Type: a.(MispAttribute).Value, + + "EventID": a.(MispAttribute).EventID, + "EventLabel": a.(MispAttribute).EventID, + "UUID": a.(MispAttribute).UUID, + "SharingGroupID": a.(MispAttribute).SharingGroupID, + "Distribution": a.(MispAttribute).Distribution, + "Category": a.(MispAttribute).Category, + "ToIDS": a.(MispAttribute).ToIDS, + "Deleted": a.(MispAttribute).Deleted, + "Comment": a.(MispAttribute).Comment, + } + + // Get tags + attributeTags := []string{} + for _, tag := range a.(MispAttribute).Tag { + attributeTags = append(attributeTags, tag.Name) + } + + entry["AttributeTags"] = strings.Join(attributeTags[:], ", ") + + // Convert timestamps to datetime + strTimestamp, err := strconv.ParseInt(a.(MispAttribute).StrTimestamp, 10, 64) + if err != nil { + return nil, debug, fmt.Errorf("Failed to parse attribute's StrTimestamp: %s", a.(MispAttribute).StrTimestamp) + } + + entry["StrTimestamp"] = time.Unix(strTimestamp, 0) + + entries = append(entries, entry) + p.generateRelations(a.(MispAttribute).Type) + } + } + + if err != nil { + return nil, debug, fmt.Errorf("Failed to search: %s", err.Error()) + } + + return entries, debug, nil +} + +func (p *plugin) generateRelations(typ string) { + if _, ok := p.types[typ]; !ok { + p.types[typ] = true + + relation := &pdk.Relation{ + From: &pdk.Node{ + ID: typ, + Group: typ, + Search: typ, + Attributes: []string{"ID", "Name", "Description", "UUID", "Comment", "Deleted", "Distribution", "SharingGroupID", "ToIDS", "FirstSeen", "LastSeen", "AttributeTags"}, + }, + To: &pdk.Node{ + ID: "EventID", + Group: "event", + Search: "event", + Attributes: []string{"EventID", "OrgcID", "OrgID", "Date", "ThreatLevelID", "Info", "Published", "EventUUID", "AttributeCount", "Analysis", "EventStrTimestamp", "EventDistribution", "ProposalEmailLock", "Locked", "StrPublishedTimestamp", "EventSharingGroupID", "Org", "Orgc", "EventTags"}, + }, + Edge: &struct { + Label string `yaml:"label"` + Attributes []string `yaml:"attributes"` + }{ + Attributes: []string{"Category", "StrTimestamp"}, + }, + } + + // Search for objects is not supported + if typ == "Label" { + relation.From.Search = "" + } + + // Set common nodes attributes for similar MISP field types + if typ == "ip-src" || typ == "ip-dst" || typ == "domain|ip" { + relation.From.Group = "ip" + relation.From.Search = "ip" + } else if typ == "domain" || typ == "hostname" { + relation.From.Group = "domain" + relation.From.Search = "domain" + } else if typ == "email" || typ == "email-src" || typ == "email-dst" || typ == "target-email" || typ == "whois-registrant-email" || typ == "dns-soa-email" { + relation.From.Group = "email" + relation.From.Search = "email" + } + + // Put backticks around uncommon field names + if strings.Contains(relation.From.Search, "-") || strings.Contains(relation.From.Search, "|") { + relation.From.Search = "`" + relation.From.Search + "`" + } + + p.source.Relations = append(p.source.Relations, relation) + } +} + +func (p *plugin) Stop() error { + + // No error to check, so return nil + return nil +} diff --git a/plugins/src/misp/misp_test.go b/plugins/src/misp/misp_test.go new file mode 100644 index 0000000..b1227d2 --- /dev/null +++ b/plugins/src/misp/misp_test.go @@ -0,0 +1,78 @@ +package main + +import ( + "testing" + + "github.com/blastrain/vitess-sqlparser/sqlparser" +) + +/* + * Test SQL conversion to the data source's expected format + */ +func TestConvert(t *testing.T) { + + // Empty plugin's instance to test + c := plugin{} + + // Pairs of SQLs and the expected results + tables := []struct { + sql string + converted []string + }{ + { + "SELECT * WHERE `domain|ip`='10.10.10.10' LIMIT 5,1", + []string{"domain|ip", "10.10.10.10"}, + }, + { + `SELECT * WHERE ip='10.10.10.10' and datetime BETWEEN '2024-05-04T11:30:14.000Z' AND '2024-06-04T11:30:14.000Z'`, + []string{"ip", "10.10.10.10", "2024-05-04T11:30:14.000Z", "2024-06-04T11:30:14.000Z"}, + }, + { + `select * where name='sarah'`, + []string{"name", "sarah"}, + }, + } + + for _, table := range tables { + // Executed by the main service + ast, err := sqlparser.Parse(table.sql) + if err != nil { + t.Errorf("Can't parse '%s': %s", table.sql, err.Error()) + continue + } + + stmt, ok := ast.(*sqlparser.Select) + if !ok { + t.Errorf("Only SELECT statement is allowed: %s", table.sql) + continue + } + + // Executed by the plugin + result, err := c.convert(stmt) + if err != nil { + t.Errorf("Can't convert '%s': %s", table.sql, err.Error()) + continue + } + + if !equal(result, table.converted) { + t.Errorf("Invalid conversion of '%s': %v, expected: %v", table.sql, result, table.converted) + } + } +} + +/* + * Check whether a and b slices contain the same elements + */ +func equal(a, b []string) bool { + if len(a) != len(b) { + return false + } + + for i, v := range a { + if v != b[i] { + return false + } + } + + return true +} diff --git a/plugins/src/misp/package.go b/plugins/src/misp/package.go new file mode 100644 index 0000000..98a56a5 --- /dev/null +++ b/plugins/src/misp/package.go @@ -0,0 +1,528 @@ +// Based on: +// https://raw.githubusercontent.com/0xrawsec/golang-misp/master/misp/misp.go + +package main + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "sort" + "strconv" + "strings" + "time" + + "github.com/0xrawsec/golang-utils/config" + "github.com/0xrawsec/golang-utils/datastructs" + "github.com/0xrawsec/golang-utils/log" + "github.com/0xrawsec/golang-utils/readers" +) + +type MispError struct { + StatusCode int + Message string +} + +func (me MispError) Error() string { + return fmt.Sprintf("MISP ERROR (HTTP %d) : %s", me.StatusCode, me.Message) +} + +type MispCon struct { + Proto string + Host string + APIKey string + Client *http.Client +} + +type MispRequest struct { + Request MispQuery `json:"request"` +} + +type MispQuery interface { + // Prepare the query and returns a JSON object in a form of a byte array + Prepare() []byte +} + +type MispObject interface{} + +type MispResponse interface { + Iter() chan MispObject +} + +type EmptyMispResponse struct{} + +// Iter : MispResponse implementation +func (emr EmptyMispResponse) Iter() chan MispObject { + c := make(chan MispObject) + close(c) + return c +} + +//////////////////////////////////////////////////////////////////////////////// +////////////////////////////////// Events ////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// + +// MispEventQuery : defines the structure of query to event search API +type MispEventQuery struct { + Value string `json:"value,omitempty"` + Type string `json:"type,omitempty"` + Category string `json:"category,omitempty"` + Org string `json:"org,omitempty"` + Tags string `json:"tags,omitempty"` + QuickFilter string `json:"quickfilter,omitempty"` + From string `json:"from,omitempty"` + To string `json:"to,omitempty"` + Last string `json:"last,omitempty"` + EventID string `json:"eventid,omitempty"` + WithAttachments string `json:"withAttachments,omitempty"` + Metadata string `json:"metadata,omitempty"` + SearchAll int8 `json:"searchall,omitempty"` +} + +// Prepare : MispQuery Implementation +func (meq MispEventQuery) Prepare() (j []byte) { + jsMeq, err := json.Marshal(MispRequest{meq}) + if err != nil { + panic(err) + } + return jsMeq +} + +// Org definition +type Org struct { + ID string `json:"id"` + Name string `json:"name"` + UUID string `json:"uuid"` +} + +// MispRelatedEvent definition +type MispRelatedEvent struct { + ID string `json:"id"` + Date string `json:"date"` + ThreatLevelID string `json:"threat_level_id"` + Info string `json:"info"` + Published bool `json:"published"` + UUID string `json:"uuid"` + Analysis string `json:"analysis"` + StrTimestamp string `json:"timestamp"` + Distribution string `json:"distribution"` + OrgID string `json:"org_id"` + OrgcID string `json:"orgc_id"` + Org Org `json:"Org"` + Orgc Org `json:"Orgc"` +} + +// Timestamp : return Time struct according to a string time +func (mre *MispRelatedEvent) Timestamp() time.Time { + sec, err := strconv.ParseInt(mre.StrTimestamp, 10, 64) + if err != nil { + panic(err) + } + return time.Unix(sec, 0) +} + +// MispTag definition +type MispTag struct { + ID string `json:"id"` + Name string `json:"name"` + Colour string `json:"colour"` + Exportable bool `json:"exportable"` + HideTag bool `json:"hide_tag"` + UserID string `json:"user_id"` + //NumericalValue string `json:"numerical_value"` // TODO: Unknown type here for now, find it + IsGalaxy bool `json:"is_galaxy"` + IsCustomGalaxy bool `json:"is_custom_galaxy"` + LocalOnly bool `json:"local_only"` + Local bool `json:"local"` + //RelationshipType `json:"relationship_type"` // TODO: Unknown type here for now, find it +} + +// EventObject definition +type EventObject struct { + ID string `json:"id"` + Name string `json:"name"` + MetaCategory string `json:"meta-category"` + Description string `json:"description"` + TemplateUUID string `json:"template_uuid"` + TemplateVersion string `json:"template_version"` + EventID string `json:"event_id"` + UUID string `json:"uuid"` + StrTimestamp string `json:"timestamp"` + Distribution string `json:"distribution"` + SharingGroupID string `json:"sharing_group_id"` + Comment string `json:"comment"` + Deleted bool `json:"deleted"` + FirstSeen string `json:"first_seen"` + LastSeen string `json:"last_seen"` + //ObjectReference string `json:"ObjectReference"` // TODO: Should be list of references + Attribute []MispAttribute `json:"Attribute"` +} + +// MispEvent definition +type MispEvent struct { + ID string `json:"id"` + OrgcID string `json:"orgc_id"` + OrgID string `json:"org_id"` + Date string `json:"date"` + ThreatLevelID string `json:"threat_level_id"` + Info string `json:"info"` + Published bool `json:"published"` + UUID string `json:"uuid"` + AttributeCount string `json:"attribute_count"` + Analysis string `json:"analysis"` + StrTimestamp string `json:"timestamp"` + Distribution string `json:"distribution"` + ProposalEmailLock bool `json:"proposal_email_lock"` + Locked bool `json:"locked"` + StrPublishedTimestamp string `json:"publish_timestamp"` + SharingGroupID string `json:"sharing_group_id"` + Org Org `json:"Org"` + Orgc Org `json:"Orgc"` + Attribute []MispAttribute `json:"Attribute"` + ShadowAttribute []MispAttribute `json:"ShadowAttribute"` + RelatedEvent []MispRelatedEvent `json:"RelatedEvent"` + Galaxy []MispRelatedEvent `json:"Galaxy"` + Object []EventObject `json:"Object"` + Tag []MispTag `json:"Tag"` +} + +// Timestamp : return Time struct according to a string time +func (me MispEvent) Timestamp() time.Time { + sec, err := strconv.ParseInt(me.StrTimestamp, 10, 64) + if err != nil { + panic(err) + } + return time.Unix(sec, 0) +} + +// PublishedTimestamp : return Time struct according to a string time +func (me MispEvent) PublishedTimestamp() time.Time { + sec, err := strconv.ParseInt(me.StrPublishedTimestamp, 10, 64) + if err != nil { + panic(err) + } + return time.Unix(sec, 0) +} + +// MispEventDict : intermediate structure to handle properly MISP API results +type MispEventDict struct { + Event MispEvent `json:"Event"` +} + +// MispEventResponse : intermediate structure to handle properly MISP API results +type MispEventResponse struct { + Response []MispEventDict `json:"response"` +} + +// Iter : MispResponse implementation +func (mer MispEventResponse) Iter() (moc chan MispObject) { + moc = make(chan MispObject) + go func() { + defer close(moc) + for _, me := range mer.Response { + moc <- me.Event + } + }() + return +} + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////// Attributes //////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// + +type MispAttributeQuery struct { + Value string `json:"value,omitempty"` + Type string `json:"type,omitempty"` + Category string `json:"category,omitempty"` + Org string `json:"org,omitempty"` + Tags string `json:"tags,omitempty"` + From string `json:"from,omitempty"` + To string `json:"to,omitempty"` + Last string `json:"last,omitempty"` + EventID string `json:"eventid,omitempty"` + UUID string `json:"uuid,omitempty"` +} + +// Prepare : MispQuery Implementation +func (maq MispAttributeQuery) Prepare() (j []byte) { + jsMaq, err := json.Marshal(MispRequest{maq}) + if err != nil { + panic(err) + } + return jsMaq +} + +// MispAttributeDict : itermediate structure to handle MISP attribute search +type MispAttributeDict struct { + Attribute []MispAttribute `json:"Attribute"` +} + +// MispAttributeResponse : API response when attribute query is done +type MispAttributeResponse struct { + Response MispAttributeDict `json:"response"` +} + +// Iter : MispResponse implementation +func (mar MispAttributeResponse) Iter() (moc chan MispObject) { + moc = make(chan MispObject) + go func() { + defer close(moc) + for _, ma := range mar.Response.Attribute { + moc <- ma + } + }() + return +} + +// MispAttribute : define structure of attribute object returned by API +type MispAttribute struct { + ID string `json:"id"` + EventID string `json:"event_id"` + UUID string `json:"uuid"` + SharingGroupID string `json:"sharing_group_id"` + StrTimestamp string `json:"timestamp"` + Distribution string `json:"distribution"` + Category string `json:"category"` + Type string `json:"type"` + Value string `json:"value"` + ToIDS bool `json:"to_ids"` + Deleted bool `json:"deleted"` + DisableCorrelation bool `json:"disable_correlation"` + ObjectID string `json:"object_id"` + ObjectRelation string `json:"object_relation"` + FirstSeen string `json:"first_seen"` + LastSeen string `json:"last_seen"` + //Galaxy + //ShadowAttribute + Comment string `json:"comment"` + Tag []MispTag `json:"Tag"` +} + +// Timestamp : return Time struct according to a string time +func (ma MispAttribute) Timestamp() time.Time { + sec, err := strconv.ParseInt(ma.StrTimestamp, 10, 64) + if err != nil { + panic(err) + } + return time.Unix(sec, 0) +} + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////// Config //////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// + +// MispConfig structure +type MispConfig struct { + Proto string `json:"protocol"` + Host string `json:"host"` + APIKey string `json:"api-key"` +} + +//////////////////////////////////////////////////////////////////////////////// +////////////////////////////// MISP Interface ////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// + +var ( + // ErrUnknownProtocol : raised when bad protocol specified + ErrUnknownProtocol = errors.New("Unknown protocol") +) + +func headerSortedKeys(d http.Header) (sk []string) { + sk = make([]string, 0, len(d)) + for k := range d { + sk = append(sk, k) + } + sort.Strings(sk) + return +} + +func logRequest(req *http.Request) { + proxyURL, err := http.ProxyFromEnvironment(req) + if err != nil { + panic(err) + } + body, _ := req.GetBody() + log.Debugf("Proxy: %s", proxyURL) + log.Debugf("%s %s", req.Method, req.URL) + log.Debug("Header:") + for _, sk := range headerSortedKeys(req.Header) { + for _, v := range req.Header[sk] { + log.Debugf(" %s: %v", sk, v) + } + } + log.Debugf("Body: %s", string(readAllOrPanic(body))) +} + +func readAllOrPanic(r io.Reader) []byte { + respBody, err := io.ReadAll(r) + if err != nil { + panic(err) + } + return respBody +} + +// LoadConfig : load a configuration file from path +// return (MispConfig) +func LoadConfig(path string) (mc MispConfig) { + conf, err := config.Load(path) + if err != nil { + panic(err) + } + mc.Proto = conf.GetRequiredString("protocol") + mc.Host = conf.GetRequiredString("host") + mc.APIKey = conf.GetRequiredString("api-key") + return +} + +// NewInsecureCon : Return a new MispCon with insecured TLS connection settings +// return (MispCon) +func NewInsecureCon(proto, host, apiKey string) MispCon { + if proto != "http" && proto != "https" { + log.Errorf("%s : only http and https protocols are allowed", ErrUnknownProtocol.Error()) + panic(ErrUnknownProtocol) + } + var noCertTransport http.RoundTripper = &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } + c := http.Client{Transport: noCertTransport} + return MispCon{proto, host, apiKey, &c} +} + +// NewCon : create a new MispCon struct +// return (MispcCon) +func NewCon(proto, host, apiKey string) MispCon { + if proto != "http" && proto != "https" { + log.Errorf("%s : only http and https protocols are allowed", ErrUnknownProtocol.Error()) + panic(ErrUnknownProtocol) + } + return MispCon{proto, host, apiKey, &http.Client{}} +} + +func (mc MispCon) buildURL(path ...string) string { + for i := range path { + path[i] = strings.TrimLeft(path[i], "/") + } + return fmt.Sprintf("%s://%s/%s", mc.Proto, mc.Host, strings.Join(path, "/")) +} + +func (mc MispCon) prepareRequest(method, url string, body io.Reader) (*http.Request, error) { + req, err := http.NewRequest(method, url, body) + req.Header.Add("Authorization", mc.APIKey) + req.Header.Add("Content-Type", "application/json") + req.Header.Add("Accept", "application/json") + req.Header.Add("User-Agent", "Graphoscope") + return req, err +} + +func (mc MispCon) postSearch(kind string, mq *MispQuery) ([]byte, error) { + fullURL := mc.buildURL(kind, "restSearch", "download") + pReq, err := mc.prepareRequest("POST", fullURL, bytes.NewReader((*mq).Prepare())) + if err != nil { + return []byte{}, err + } + if err != nil { + return []byte{}, err + } + logRequest(pReq) + pResp, err := mc.Client.Do(pReq) + if err != nil { + return []byte{}, err + } + defer pResp.Body.Close() + + respBody := readAllOrPanic(pResp.Body) + switch pResp.StatusCode { + case 200: + return respBody, err + default: + return []byte{}, MispError{pResp.StatusCode, string(respBody)} + } +} + +// Search : Issue a search and return a MispObject +// @mq : a struct implementing MispQuery interface +// return (MispObject, error) +func (mc MispCon) Search(mq MispQuery) (MispResponse, error) { + switch mq.(type) { + case MispAttributeQuery: + mar := MispAttributeResponse{} + bResp, err := mc.postSearch("attributes", &mq) + if err != nil { + log.Debugf("Error: %s", err) + return EmptyMispResponse{}, err + } + err = json.Unmarshal(bResp, &mar) + if err != nil { + log.Debug(string(bResp)) + return mar, err + } + return mar, nil + + case MispEventQuery: + mer := MispEventResponse{} + bResp, err := mc.postSearch("events", &mq) + if err != nil { + log.Debugf("Error: %s", err) + return EmptyMispResponse{}, err + } + + err = json.Unmarshal(bResp, &mer) + if err != nil { + log.Debug(string(bResp)) + return mer, err + } + return mer, nil + } + return EmptyMispResponse{}, fmt.Errorf("Empty Response") +} + +// TextExport text export API wrapper https:///attributes/text/download/ +// The wrapper takes care of removing the duplicated entries +// @flags: the list of flags to use for the query +func (mc MispCon) TextExport(flags ...string) (out []string, err error) { + path := make([]string, 0) + path = append(path, "attributes", "text", "download") + path = append(path, flags...) + + url := mc.buildURL(path...) + + out = make([]string, 0) + + pReq, err := mc.prepareRequest("GET", url, new(bytes.Buffer)) + if err != nil { + return + } + logRequest(pReq) + pResp, err := mc.Client.Do(pReq) + if err != nil { + return + } + defer pResp.Body.Close() + switch pResp.StatusCode { + case 200: + // used to remove duplicates + marked := datastructs.NewSyncedSet() + for line := range readers.Readlines(pResp.Body) { + txt := string(line) + if !marked.Contains(txt) { + out = append(out, txt) + } + marked.Add(txt) + } + default: + return out, MispError{pResp.StatusCode, string(readAllOrPanic(pResp.Body))} + } + return +} diff --git a/plugins/src/misp/plugin.go b/plugins/src/misp/plugin.go new file mode 100644 index 0000000..1751d78 --- /dev/null +++ b/plugins/src/misp/plugin.go @@ -0,0 +1,33 @@ +package main + +import ( + "github.com/cert-lv/graphoscope/pdk" +) + +/* + * Export symbols + */ +var ( + Name = "misp" + Version = "1.0.0" + Plugin plugin +) + +/* + * Structure to be imported by the core as a plugin + */ +type plugin struct { + + // Inherit default configuration fields + source *pdk.Source + + // Custom fields + protocol string + host string + apiKey string + caCertPath string + certPath string + keyPath string + types map[string]bool + limit int +} diff --git a/plugins/src/misp/select.go b/plugins/src/misp/select.go new file mode 100644 index 0000000..ea1673e --- /dev/null +++ b/plugins/src/misp/select.go @@ -0,0 +1,218 @@ +package main + +import ( + "errors" + "fmt" + "strings" + + "github.com/blastrain/vitess-sqlparser/sqlparser" +) + +/* + * Handle single "field operator value" expression. + * + * Receives: + * expr - SQL expression to process + */ +func handleSelectWhereComparisonExpr(expr *sqlparser.Expr) ([]string, error) { + comparisonExpr := (*expr).(*sqlparser.ComparisonExpr) + colName, ok := comparisonExpr.Left.(*sqlparser.ColName) + + if !ok { + return nil, errors.New("Invalid comparison expression, the left must be a column name") + } + + colNameStr := sqlparser.String(colName) + colNameStr = strings.Replace(colNameStr, "`", "", -1) + rightIntf, err := buildComparisonExprRightStr(comparisonExpr.Right) + if err != nil { + return nil, err + } + + field := []string{} + + switch comparisonExpr.Operator { + case "=": + field = []string{colNameStr, fmt.Sprintf("%s", rightIntf)} + default: + return nil, errors.New("'=' operator is supported only") + } + + return field, nil +} + +/* + * Handle "expression AND expression". + * + * Receives: + * expr - SQL expression to process + */ +func handleSelectWhereAndExpr(expr *sqlparser.Expr) ([]string, error) { + andExpr := (*expr).(*sqlparser.AndExpr) + leftExpr := andExpr.Left + rightExpr := andExpr.Right + + leftStr, err := handleSelectWhere(&leftExpr, false, expr) + if err != nil { + return nil, err + } + + rightStr, err := handleSelectWhere(&rightExpr, false, expr) + if err != nil { + return nil, err + } + + fields := append(leftStr, rightStr...) + return fields, nil +} + +/* + * Handle "BETWEEN a AND b". + * + * Receives: + * expr - SQL expression to process + */ +func handleSelectWhereBetweenExpr(expr *sqlparser.Expr) ([]string, error) { + rangeCond := (*expr).(*sqlparser.RangeCond) + + var from string + var to string + + // Prepare a 'From' value + switch expr := rangeCond.From.(type) { + case *sqlparser.SQLVal: + switch expr.Type { + case sqlparser.IntVal, sqlparser.FloatVal, sqlparser.StrVal: + from = string(expr.Val) + default: + return nil, fmt.Errorf("Invalid BETWEEN 'from' value: %v (type %v)", string(expr.Val), expr.Type) + } + default: + return nil, fmt.Errorf("Invalid BETWEEN 'from' value: %v", strings.Trim(sqlparser.String(rangeCond.From), "'")) + } + + // Prepare a 'To' value + switch expr := rangeCond.To.(type) { + case *sqlparser.SQLVal: + switch expr.Type { + case sqlparser.IntVal, sqlparser.FloatVal, sqlparser.StrVal: + to = string(expr.Val) + default: + return nil, fmt.Errorf("Invalid BETWEEN 'to' value: %v (type %v)", string(expr.Val), expr.Type) + } + default: + return nil, fmt.Errorf("Invalid BETWEEN 'to' value: %v", strings.Trim(sqlparser.String(rangeCond.To), "'")) + } + + fields := []string{from, to} + return fields, nil +} + +/* + * Handle top level or groups of expressions. + * + * Receives: + * expr - SQL expression to process + * topLevel - whether it's a top level expression + * parent - container of the expression + */ +func handleSelectWhereParenExpr(expr *sqlparser.Expr, topLevel bool, parent *sqlparser.Expr) ([]string, error) { + parentBoolExpr := (*expr).(*sqlparser.ParenExpr) + boolExpr := parentBoolExpr.Expr + + // If parent is the top level, bool must is needed + var isThisTopLevel = false + if topLevel { + isThisTopLevel = true + } + + return handleSelectWhere(&boolExpr, isThisTopLevel, parent) +} + +func buildNestedFuncStrValue(nestedFunc *sqlparser.FuncExpr) (string, error) { + return "", errors.New("Unsupported function: " + nestedFunc.Name.String()) +} + +/* + * Check the right part of the expression + * and return its value of specific type. + * + * Receives SQL expression to process + */ +func buildComparisonExprRightStr(expr sqlparser.Expr) (interface{}, error) { + var rightStr string + var err error + + switch expr := expr.(type) { + case *sqlparser.SQLVal: + // Use string value type only + rightStr = sqlparser.String(expr) + rightStr = strings.Trim(rightStr, "'") + + case *sqlparser.BoolVal, sqlparser.BoolVal: + rightStr = sqlparser.String(expr) + + case *sqlparser.GroupConcatExpr: + return nil, errors.New("group_concat not supported") + + case *sqlparser.FuncExpr: + // Parse nested + //funcExpr := expr.(*sqlparser.FuncExpr) + //rightStr, err = buildNestedFuncStrValue(funcExpr) + rightStr, err = buildNestedFuncStrValue(expr) + if err != nil { + return nil, err + } + + case *sqlparser.ColName: + if sqlparser.String(expr) == "exist" { + return nil, errors.New("'exist' expression currently not supported") + } + return nil, errors.New("Column name on the right side of compare operator is not supported") + + case sqlparser.ValTuple: + rightStr = sqlparser.String(expr) + + default: + return nil, fmt.Errorf("Unexpected SQL expression right part's type: %T", expr) + } + + return rightStr, err +} + +/* + * Handle WHERE statement. + * + * Receives: + * expr - SQL expression to process + * topLevel - whether it's a top level expression + * parent - container of the expression + */ +func handleSelectWhere(expr *sqlparser.Expr, topLevel bool, parent *sqlparser.Expr) ([]string, error) { + if expr == nil { + return nil, errors.New("SQL expression cannot be nil here") + } + + switch (*expr).(type) { + case *sqlparser.ComparisonExpr: + return handleSelectWhereComparisonExpr(expr) + + // Needed for datetime range + case *sqlparser.AndExpr: + return handleSelectWhereAndExpr(expr) + + case *sqlparser.IsExpr: + return nil, errors.New("'is' expression currently not supported") + + case *sqlparser.NotExpr: + return nil, errors.New("'not' expression currently not supported") + + case *sqlparser.RangeCond: + return handleSelectWhereBetweenExpr(expr) + + case *sqlparser.ParenExpr: + return handleSelectWhereParenExpr(expr, topLevel, parent) + } + + return nil, fmt.Errorf("Unexpected SQL expression type received: %T", *expr) +}