From ff13812503611fee29d180e4eb0705a335ae84af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Romain=20Tarti=C3=A8re?= Date: Wed, 19 Jun 2024 10:31:35 -1000 Subject: [PATCH] Add support for ignoring IPs by ASN in `riemann-http` Monitoring a website behind a Content Delivery Network (CDN) may lead to flapping metrics when the short-lived IP addresses where the service is accessible change. Allow to provide a list of Autonomous System Numbers (ASN) that we can ignore for well-known CDN service providers. Use the MaxMind ASN database provided by the user for IP lookups. This is not a hard dependency as no ASN filtering is done by default, so only add this dependency for testing and assume the end-user will handle the soft requirement on his own if they want to do filter-out some ASN. --- .gitignore | 1 + Gemfile | 1 + lib/riemann/tools/http_check.rb | 25 +++++ .../test-asn/GeoLite2-ASN-Blocks-IPv4.csv | 4 + .../test-asn/GeoLite2-ASN-Blocks-IPv6.csv | 4 + spec/fixtures/test-asn/README.md | 19 ++++ spec/fixtures/test-asn/go.mod | 11 +++ spec/fixtures/test-asn/go.sum | 8 ++ spec/fixtures/test-asn/main.go | 88 ++++++++++++++++++ spec/fixtures/test-asn/test-asn.mmdb | Bin 0 -> 2690 bytes spec/riemann/tools/http_check_spec.rb | 24 +++++ 11 files changed, 185 insertions(+) create mode 100644 spec/fixtures/test-asn/GeoLite2-ASN-Blocks-IPv4.csv create mode 100644 spec/fixtures/test-asn/GeoLite2-ASN-Blocks-IPv6.csv create mode 100644 spec/fixtures/test-asn/README.md create mode 100644 spec/fixtures/test-asn/go.mod create mode 100644 spec/fixtures/test-asn/go.sum create mode 100644 spec/fixtures/test-asn/main.go create mode 100644 spec/fixtures/test-asn/test-asn.mmdb diff --git a/.gitignore b/.gitignore index 33418d60..6c02235b 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ pkg/ .*.swp *.log lib/riemann/tools/*_parser.tab.rb +spec/fixtures/test-asn/test-asn diff --git a/Gemfile b/Gemfile index 28b89af7..c853a079 100644 --- a/Gemfile +++ b/Gemfile @@ -6,6 +6,7 @@ source 'https://rubygems.org' gemspec gem 'github_changelog_generator' +gem 'maxmind-geoip2' gem 'racc' gem 'rake' gem 'rspec' diff --git a/lib/riemann/tools/http_check.rb b/lib/riemann/tools/http_check.rb index c1feba9c..c5e7814c 100644 --- a/lib/riemann/tools/http_check.rb +++ b/lib/riemann/tools/http_check.rb @@ -32,6 +32,8 @@ class HttpCheck opt :resolvers, 'Run this number of resolver threads', short: :none, type: :integer, default: 5 opt :workers, 'Run this number of worker threads', short: :none, type: :integer, default: 20 opt :user_agent, 'User-Agent header for HTTP requests', short: :none, default: "#{File.basename($PROGRAM_NAME)}/#{Riemann::Tools::VERSION} (+https://github.com/riemann/riemann-tools)" + opt :ignored_asn, 'Ignore addresses belonging to these ASN', short: :none, type: :integers, default: [] + opt :geoip_asn_database, 'Path to the GeoIP ASN database', short: :none, default: '/usr/share/GeoIP/GeoLite2-ASN.mmdb' def initialize super @@ -60,6 +62,14 @@ def initialize end end + if opts[:ignored_asn].any? + addresses.reject! do |address| + address_belongs_to_ignored_asn?(address) + end + end + + next if addresses.empty? + @work_queue.push([uri, addresses]) end end @@ -77,6 +87,21 @@ def initialize end end + def address_belongs_to_ignored_asn?(address) + begin + require 'maxmind/geoip2' + rescue LoadError + raise StandardError, 'MaxMind::GeoIP2 is not available. Please install the maxmind-geoip2 gem for filtering by ASN.' + end + + @reader ||= MaxMind::GeoIP2::Reader.new(database: opts[:geoip_asn_database]) + asn = @reader.asn(address.to_s) + + opts[:ignored_asn].include?(asn&.autonomous_system_number) + rescue MaxMind::GeoIP2::AddressNotFoundError + false + end + # Under normal operation, we have a single instance of this class for the # lifetime of the process. But when testing, we create a new instance # for each test, each with its resolvers and worker threads. The test diff --git a/spec/fixtures/test-asn/GeoLite2-ASN-Blocks-IPv4.csv b/spec/fixtures/test-asn/GeoLite2-ASN-Blocks-IPv4.csv new file mode 100644 index 00000000..fdbfd6c5 --- /dev/null +++ b/spec/fixtures/test-asn/GeoLite2-ASN-Blocks-IPv4.csv @@ -0,0 +1,4 @@ +network,autonomous_system_number,autonomous_system_organization +1.1.1.0/24,64512,FOO +2.2.2.0/24,64513,BAR +3.3.3.0/24,64514,BAZ diff --git a/spec/fixtures/test-asn/GeoLite2-ASN-Blocks-IPv6.csv b/spec/fixtures/test-asn/GeoLite2-ASN-Blocks-IPv6.csv new file mode 100644 index 00000000..457789a6 --- /dev/null +++ b/spec/fixtures/test-asn/GeoLite2-ASN-Blocks-IPv6.csv @@ -0,0 +1,4 @@ +network,autonomous_system_number,autonomous_system_organization +2001:1::/20,64512,FOO +2001:2::/20,64513,BAR +2001:3::/20,64514,BAZ diff --git a/spec/fixtures/test-asn/README.md b/spec/fixtures/test-asn/README.md new file mode 100644 index 00000000..19e95060 --- /dev/null +++ b/spec/fixtures/test-asn/README.md @@ -0,0 +1,19 @@ +# test-asn + +This is a copy of the asn-writer example from [MaxMind's `mmdbwriter` repository](https://github.com/maxmind/mmdbwriter), with some tooling to build the `test-asn.mmdb` file from the `GeoLite2-ASN-Blocks-IPv4.csv` and `GeoLite2-ASN-Blocks-IPv6.csv` files. + +## Usage + +Adjsut the `.cvs` files, then (re)generate `test-asn.mmdb` with: + +```sh +go get +go build +./test-asn +``` + +## Note + +The `mmdbwriter` code does not allow to use private neworks nor networks reserved for documentation. +The test ASN database therefore contains (obviously incorrect) information about *real* networks. +It goes without saying, but I will still say it: do not use this database for anything else than testing the riemann-tools. diff --git a/spec/fixtures/test-asn/go.mod b/spec/fixtures/test-asn/go.mod new file mode 100644 index 00000000..dd381136 --- /dev/null +++ b/spec/fixtures/test-asn/go.mod @@ -0,0 +1,11 @@ +module test-asn + +go 1.21 + +require github.com/maxmind/mmdbwriter v1.0.0 + +require ( + github.com/oschwald/maxminddb-golang v1.12.0 // indirect + go4.org/netipx v0.0.0-20220812043211-3cc044ffd68d // indirect + golang.org/x/sys v0.10.0 // indirect +) diff --git a/spec/fixtures/test-asn/go.sum b/spec/fixtures/test-asn/go.sum new file mode 100644 index 00000000..a646df95 --- /dev/null +++ b/spec/fixtures/test-asn/go.sum @@ -0,0 +1,8 @@ +github.com/maxmind/mmdbwriter v1.0.0 h1:bieL4P6yaYaHvbtLSwnKtEvScUKKD6jcKaLiTM3WSMw= +github.com/maxmind/mmdbwriter v1.0.0/go.mod h1:noBMCUtyN5PUQ4H8ikkOvGSHhzhLok51fON2hcrpKj8= +github.com/oschwald/maxminddb-golang v1.12.0 h1:9FnTOD0YOhP7DGxGsq4glzpGy5+w7pq50AS6wALUMYs= +github.com/oschwald/maxminddb-golang v1.12.0/go.mod h1:q0Nob5lTCqyQ8WT6FYgS1L7PXKVVbgiymefNwIjPzgY= +go4.org/netipx v0.0.0-20220812043211-3cc044ffd68d h1:ggxwEf5eu0l8v+87VhX1czFh8zJul3hK16Gmruxn7hw= +go4.org/netipx v0.0.0-20220812043211-3cc044ffd68d/go.mod h1:tgPU4N2u9RByaTN3NC2p9xOzyFpte4jYwsIIRF7XlSc= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/spec/fixtures/test-asn/main.go b/spec/fixtures/test-asn/main.go new file mode 100644 index 00000000..474ce331 --- /dev/null +++ b/spec/fixtures/test-asn/main.go @@ -0,0 +1,88 @@ +// asn-writer is an example of how to create an ASN MaxMind DB file from the +// GeoLite2 ASN CSVs. You must have the CSVs in the current working directory. +package main + +import ( + "encoding/csv" + "io" + "log" + "net" + "os" + "strconv" + + "github.com/maxmind/mmdbwriter" + "github.com/maxmind/mmdbwriter/mmdbtype" +) + +func main() { + writer, err := mmdbwriter.New( + mmdbwriter.Options{ + DatabaseType: "GeoLite2-ASN", + RecordSize: 24, + }, + ) + if err != nil { + log.Fatal(err) + } + + for _, file := range []string{"GeoLite2-ASN-Blocks-IPv4.csv", "GeoLite2-ASN-Blocks-IPv6.csv"} { + fh, err := os.Open(file) + if err != nil { + log.Fatal(err) + } + + r := csv.NewReader(fh) + + // first line + r.Read() + + for { + row, err := r.Read() + if err == io.EOF { + break + } + if err != nil { + log.Fatal(err) + } + + if len(row) != 3 { + log.Fatalf("unexpected CSV rows: %v", row) + } + + _, network, err := net.ParseCIDR(row[0]) + if err != nil { + log.Fatal(err) + } + + asn, err := strconv.Atoi(row[1]) + if err != nil { + log.Fatal(err) + } + + record := mmdbtype.Map{} + + if asn != 0 { + record["autonomous_system_number"] = mmdbtype.Uint32(asn) + } + + if row[2] != "" { + record["autonomous_system_organization"] = mmdbtype.String(row[2]) + } + + err = writer.Insert(network, record) + if err != nil { + log.Fatal(err) + } + } + } + + fh, err := os.Create("test-asn.mmdb") + if err != nil { + log.Fatal(err) + } + + _, err = writer.WriteTo(fh) + if err != nil { + log.Fatal(err) + } +} diff --git a/spec/fixtures/test-asn/test-asn.mmdb b/spec/fixtures/test-asn/test-asn.mmdb new file mode 100644 index 0000000000000000000000000000000000000000..39daad74dad95d5fd657f133437ecf9972608c6e GIT binary patch literal 2690 zcmZY91#}cw0EOZELU4D7;O_2(pb>&QER)?ySjg^#6-a`+OWj+jx76Jwb)oJ`X=zJa zN?obU{Om$y&z$evci)>gGrMzEGBR7F6q&tJIhh4gD!DvXz=~K&roB{|Tm`FQHLQ*` zuqM{R+E@qcVm+*n4X`0L!p7JHn_@F;jxDeyreP~=jcu?kw!`+=0Xt$R?2KKoD|W-~ z*aLfFFYJwdurKz*{x|>!;vgK1LvSb#!{ImrM`AjT!VDaZV{j~v!|^x)GcgOZF$X7N zE>6P9G7Y6E0-%G6F^(KK*}cH-VhrQB43~R)$3Ce<+9s9KufUbK3RmMAPjCOV(mKL= z+<+T#6K=*Wp5FScG9MAP;||=3yKpz|@$}Z8^bF}thG*f~cn+S6=XrYT&nFy{E=)ET zC4x75iFB!SrF21(Ns<}rL+`b>I4dRuyu z<|%v{pTTGGIeZ=u;tTj9zJxF1E7DuitK`@4b$kQg^!(?ccM?Or%Yyf$_X!{1hj_@- zJ#_MwI4ph4@Dn_eoc%NzK2L_D(ih46rSz4Tbu7^&_B<|qL;o#)hu=#-6Mm3>B>d!g z>d-IJuk^nqO8w68g!D(U`IF`^_m7{(Bzlt~Q!bg^J8;td2FX zCe}iCtJ;J*SXZVVp*}XiQ;*PyVPkB9O|cm^#}?QU)36n`#x~d%+j(lCOb3P?u@iR2 zF4z^jVR!6-J+T+|#y;2=`(b|^fCF(54#puk6o=t(9DyS-9Ya)K7RTXuoPe2_ zh1r;c6EPPj;bfeGQ*jzj#~C;iXW?v|gL82n&c~C#l#lFwvF^~qJoIA#ZOq34T!cYf zEVG%gM5d4s!Z12mB(p+hsmw8%2uA5+7{_I}9E-68OFg}x-n~Qj^p!HJ2yR|YSR=DG z*{mb1#|<(Y37e496Z^P%D;sXZ?YIMX;x3up1b5}0M9{xKhi6&wm=ktFPCV+1mPBKA z$QO=>^6W_23C%bEpL8MxRybH{#ez;aV{C5j;WSfr!lX6N7?nPSoI;*+;NbBbt2ieZ z4s`cBp`&y2f?+FC;>&j;AuHw!S&N;BZ