From d0b617ffa29ab4bdee98c2c55bf400ca7600111e Mon Sep 17 00:00:00 2001 From: Thomas Casteleyn Date: Tue, 10 Oct 2023 16:47:19 +0200 Subject: [PATCH 1/5] feat(outputs.bigquery): Project no longer required field --- plugins/outputs/bigquery/README.md | 4 +--- plugins/outputs/bigquery/bigquery.go | 18 ++++++++++-------- plugins/outputs/bigquery/sample.conf | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/plugins/outputs/bigquery/README.md b/plugins/outputs/bigquery/README.md index 4b59efcb6a4b5..d65527161bc90 100644 --- a/plugins/outputs/bigquery/README.md +++ b/plugins/outputs/bigquery/README.md @@ -26,7 +26,7 @@ See the [CONFIGURATION.md][CONFIGURATION.md] for more details. credentials_file = "/path/to/service/account/key.json" ## Google Cloud Platform Project - project = "my-gcp-project" + # project = "*detect-project-id*" ## The namespace for the metric descriptor dataset = "telegraf" @@ -38,8 +38,6 @@ See the [CONFIGURATION.md][CONFIGURATION.md] for more details. # replace_hyphen_to = "_" ``` -Requires `project` to specify where BigQuery entries will be persisted. - Requires `dataset` to specify under which BigQuery dataset the corresponding metrics tables reside. diff --git a/plugins/outputs/bigquery/bigquery.go b/plugins/outputs/bigquery/bigquery.go index a115633005b49..6a0da84d3623a 100644 --- a/plugins/outputs/bigquery/bigquery.go +++ b/plugins/outputs/bigquery/bigquery.go @@ -4,6 +4,7 @@ package bigquery import ( "context" _ "embed" + "errors" "fmt" "reflect" "strings" @@ -45,21 +46,21 @@ func (*BigQuery) SampleConfig() string { return sampleConfig } -func (s *BigQuery) Connect() error { - if s.Project == "" { - return fmt.Errorf("Project is a required field for BigQuery output") - } - +func (s *BigQuery) Init() error { if s.Dataset == "" { - return fmt.Errorf("Dataset is a required field for BigQuery output") + return errors.New(`"dataset" is required`) } + s.warnedOnHyphens = make(map[string]bool) + + return nil +} + +func (s *BigQuery) Connect() error { if s.client == nil { return s.setUpDefaultClient() } - s.warnedOnHyphens = make(map[string]bool) - return nil } @@ -222,6 +223,7 @@ func (s *BigQuery) Close() error { func init() { outputs.Add("bigquery", func() telegraf.Output { return &BigQuery{ + Project: bigquery.DetectProjectID, Timeout: defaultTimeout, ReplaceHyphenTo: "_", } diff --git a/plugins/outputs/bigquery/sample.conf b/plugins/outputs/bigquery/sample.conf index 43aa9933cb5b0..7e9384b14045b 100644 --- a/plugins/outputs/bigquery/sample.conf +++ b/plugins/outputs/bigquery/sample.conf @@ -4,7 +4,7 @@ credentials_file = "/path/to/service/account/key.json" ## Google Cloud Platform Project - project = "my-gcp-project" + # project = "*detect-project-id*" ## The namespace for the metric descriptor dataset = "telegraf" From ca5191b217de079f0524d23ad6c202ac6bdce5d8 Mon Sep 17 00:00:00 2001 From: Thomas Casteleyn Date: Tue, 10 Oct 2023 16:48:12 +0200 Subject: [PATCH 2/5] chore(deps): Bump cloud.google.com/go/bigquery from 1.51.0 to 1.56.0 --- go.mod | 16 ++++++++-------- go.sum | 44 ++++++++++++++++++++------------------------ 2 files changed, 28 insertions(+), 32 deletions(-) diff --git a/go.mod b/go.mod index 20a0a1d37e6f6..d15f92dd49264 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/influxdata/telegraf go 1.21 require ( - cloud.google.com/go/bigquery v1.52.0 + cloud.google.com/go/bigquery v1.56.0 cloud.google.com/go/monitoring v1.15.1 cloud.google.com/go/pubsub v1.33.0 cloud.google.com/go/storage v1.30.1 @@ -199,8 +199,8 @@ require ( golang.org/x/text v0.13.0 golang.zx2c4.com/wireguard/wgctrl v0.0.0-20211230205640-daad0b7ba671 gonum.org/v1/gonum v0.13.0 - google.golang.org/api v0.134.0 - google.golang.org/genproto/googleapis/api v0.0.0-20230717213848-3f92550aa753 + google.golang.org/api v0.139.0 + google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5 google.golang.org/grpc v1.57.0 google.golang.org/protobuf v1.31.0 gopkg.in/gorethink/gorethink.v3 v3.0.5 @@ -215,8 +215,8 @@ require ( ) require ( - cloud.google.com/go v0.110.4 // indirect - cloud.google.com/go/compute v1.22.0 // indirect + cloud.google.com/go v0.110.6 // indirect + cloud.google.com/go/compute v1.23.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v1.1.1 // indirect code.cloudfoundry.org/clock v1.0.0 // indirect @@ -324,7 +324,7 @@ require ( github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/s2a-go v0.1.4 // indirect + github.com/google/s2a-go v0.1.7 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/grid-x/serial v0.0.0-20211107191517-583c7356b3aa // indirect @@ -466,8 +466,8 @@ require ( golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect golang.zx2c4.com/wireguard v0.0.0-20211209221555-9c9e7e272434 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230717213848-3f92550aa753 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230720185612-659f7aaaa771 // indirect + google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect gopkg.in/errgo.v1 v1.0.1 // indirect gopkg.in/fatih/pool.v2 v2.0.0 // indirect gopkg.in/fsnotify.v1 v1.4.7 // indirect diff --git a/go.sum b/go.sum index cf1e63c3b202f..5cd29ce02a9fe 100644 --- a/go.sum +++ b/go.sum @@ -5,23 +5,23 @@ cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6A cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.110.4 h1:1JYyxKMN9hd5dR2MYTPWkGUgcoxVVhg0LKNKEo0qvmk= -cloud.google.com/go v0.110.4/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= +cloud.google.com/go v0.110.6 h1:8uYAkj3YHTP/1iwReuHPxLSbdcyc+dSBbzFMrVwDR6Q= +cloud.google.com/go v0.110.6/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.52.0 h1:JKLNdxI0N+TIUWD6t9KN646X27N5dQWq9dZbbTWZ8hc= -cloud.google.com/go/bigquery v1.52.0/go.mod h1:3b/iXjRQGU4nKa87cXeg6/gogLjO8C6PmuM8i5Bi/u4= -cloud.google.com/go/compute v1.22.0 h1:cB8R6FtUtT1TYGl5R3xuxnW6OUIc/DrT2aiR16TTG7Y= -cloud.google.com/go/compute v1.22.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= +cloud.google.com/go/bigquery v1.56.0 h1:LHIc9E7Kw+ftFpQFKzZYBB88IAFz7qONawXXx0F3QBo= +cloud.google.com/go/bigquery v1.56.0/go.mod h1:KDcsploXTEY7XT3fDQzMUZlpQLHzE4itubHrnmhUrZA= +cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= +cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/datacatalog v1.14.1 h1:cFPBt8V5V2T3mu/96tc4nhcMB+5cYcpwjBfn79bZDI8= -cloud.google.com/go/datacatalog v1.14.1/go.mod h1:d2CevwTG4yedZilwe+v3E3ZBDRMobQfSG/a6cCCN5R4= +cloud.google.com/go/datacatalog v1.16.0 h1:qVeQcw1Cz93/cGu2E7TYUPh8Lz5dn5Ws2siIuQ17Vng= +cloud.google.com/go/datacatalog v1.16.0/go.mod h1:d2CevwTG4yedZilwe+v3E3ZBDRMobQfSG/a6cCCN5R4= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/iam v1.1.1 h1:lW7fzj15aVIXYHREOqjRBV9PsH0Z6u8Y46a1YGvQP4Y= cloud.google.com/go/iam v1.1.1/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= -cloud.google.com/go/kms v1.12.1 h1:xZmZuwy2cwzsocmKDOPu4BL7umg8QXagQx6fKVmf45U= -cloud.google.com/go/kms v1.12.1/go.mod h1:c9J991h5DTl+kg7gi3MYomh12YEENGrf48ee/N/2CDM= +cloud.google.com/go/kms v1.15.0 h1:xYl5WEaSekKYN5gGRyhjvZKM22GVBBCzegGNVPy+aIs= +cloud.google.com/go/kms v1.15.0/go.mod h1:c9J991h5DTl+kg7gi3MYomh12YEENGrf48ee/N/2CDM= cloud.google.com/go/longrunning v0.5.1 h1:Fr7TXftcqTudoyRJa113hyaqlGdiBQkp0Gq7tErFDWI= cloud.google.com/go/longrunning v0.5.1/go.mod h1:spvimkwdz6SPWKEt/XBij79E9fiTkHSQl/fRUUQJYJc= cloud.google.com/go/monitoring v1.15.1 h1:65JhLMd+JiYnXr6j5Z63dUYCuOg770p8a/VC+gil/58= @@ -385,7 +385,6 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= @@ -492,7 +491,6 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A= @@ -703,8 +701,8 @@ github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8 h1:n6vlPhxsA+BW/XsS5+ github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA= github.com/google/protobuf v3.11.4+incompatible/go.mod h1:lUQ9D1ePzbH2PrIS7ob/bjm9HXyH5WHB0Akwh7URreM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= -github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -1628,7 +1626,6 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211202192323-5770296d904e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= @@ -1950,8 +1947,8 @@ google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.134.0 h1:ktL4Goua+UBgoP1eL1/60LwZJqa1sIzkLmvoR3hR6Gw= -google.golang.org/api v0.134.0/go.mod h1:sjRL3UnjTx5UqNQS9EWr9N8p7xbHpy1k0XGRLCf3Spk= +google.golang.org/api v0.139.0 h1:A1TrCPgMmOiYu0AiNkvQIpIx+D8blHTDcJ5EogkP7LI= +google.golang.org/api v0.139.0/go.mod h1:CVagp6Eekz9CjGZ718Z+sloknzkDJE7Vc1Ckj9+viBk= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1975,12 +1972,12 @@ google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEY google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20230717213848-3f92550aa753 h1:+VoAg+OKmWaommL56xmZSE2sUK8A7m6SUO7X89F2tbw= -google.golang.org/genproto v0.0.0-20230717213848-3f92550aa753/go.mod h1:iqkVr8IRpZ53gx1dEnWlCUIEwDWqWARWrbzpasaTNYM= -google.golang.org/genproto/googleapis/api v0.0.0-20230717213848-3f92550aa753 h1:lCbbUxUDD+DiXx9Q6F/ttL0aAu7N2pz8XnmMm8ZW4NE= -google.golang.org/genproto/googleapis/api v0.0.0-20230717213848-3f92550aa753/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230720185612-659f7aaaa771 h1:Z8qdAF9GFsmcUuWQ5KVYIpP3PCKydn/YKORnghIalu4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230720185612-659f7aaaa771/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= +google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 h1:L6iMMGrtzgHsWofoFcihmDEMYeDR9KN/ThbPWGrh++g= +google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5/go.mod h1:oH/ZOT02u4kWEp7oYBGYFFkCdKS/uYR9Z7+0/xuuFp8= +google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5 h1:nIgk/EEq3/YlnmVVXVnm14rC2oxgs1o0ong4sD/rd44= +google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5/go.mod h1:5DZzOUPCLYL3mNkQ0ms0F3EuUNZ7py1Bqeq6sxzI7/Q= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1997,7 +1994,6 @@ google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA5 google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= From 5f5db92faed37b2fbb14103b121b2f21156b804b Mon Sep 17 00:00:00 2001 From: Thomas Casteleyn Date: Tue, 10 Oct 2023 16:51:12 +0200 Subject: [PATCH 3/5] test(outputs.bigquery): Reorganise and cleanup tests --- plugins/outputs/bigquery/bigquery_test.go | 169 ++++++++++++---------- 1 file changed, 95 insertions(+), 74 deletions(-) diff --git a/plugins/outputs/bigquery/bigquery_test.go b/plugins/outputs/bigquery/bigquery_test.go index dd029f23f7ff6..57f668c0eaf59 100644 --- a/plugins/outputs/bigquery/bigquery_test.go +++ b/plugins/outputs/bigquery/bigquery_test.go @@ -5,23 +5,19 @@ import ( "encoding/json" "net/http" "net/http/httptest" - "strings" "testing" "time" "cloud.google.com/go/bigquery" - "github.com/influxdata/telegraf/config" "github.com/influxdata/telegraf/testutil" "github.com/stretchr/testify/require" "google.golang.org/api/option" ) const ( - successfulResponse = "{\"kind\": \"bigquery#tableDataInsertAllResponse\"}" + successfulResponse = `{"kind": "bigquery#tableDataInsertAllResponse"}` ) -var testingHost string -var testDuration = config.Duration(5 * time.Second) var receivedBody map[string]json.RawMessage type Row struct { @@ -30,56 +26,121 @@ type Row struct { Value float64 `json:"value"` } +func TestInit(t *testing.T) { + tests := []struct { + name string + errorString string + plugin *BigQuery + }{ + { + name: "dataset is not set", + errorString: `"dataset" is required`, + plugin: &BigQuery{}, + }, + { + name: "valid config", + plugin: &BigQuery{ + Dataset: "test-dataset", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.errorString != "" { + require.EqualError(t, tt.plugin.Init(), tt.errorString) + } else { + require.NoError(t, tt.plugin.Init()) + } + }) + } +} + +func TestMetricToTable(t *testing.T) { + tests := []struct { + name string + replaceHyphenTo string + metricName string + expectedTable string + }{ + { + name: "no rename", + replaceHyphenTo: "_", + metricName: "test", + expectedTable: "test", + }, + { + name: "default config", + replaceHyphenTo: "_", + metricName: "table-with-hyphens", + expectedTable: "table_with_hyphens", + }, + { + name: "custom hypens", + replaceHyphenTo: "*", + metricName: "table-with-hyphens", + expectedTable: "table*with*hyphens", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := &BigQuery{ + Dataset: "test-dataset", + ReplaceHyphenTo: tt.replaceHyphenTo, + Log: testutil.Logger{}, + } + require.NoError(t, b.Init()) + + require.Equal(t, tt.expectedTable, b.metricToTable(tt.metricName)) + if tt.metricName != tt.expectedTable { + require.Contains(t, b.warnedOnHyphens, tt.metricName) + require.True(t, b.warnedOnHyphens[tt.metricName]) + } else { + require.NotContains(t, b.warnedOnHyphens, tt.metricName) + } + }) + } +} + func TestConnect(t *testing.T) { srv := localBigQueryServer(t) - testingHost = strings.ReplaceAll(srv.URL, "http://", "") defer srv.Close() b := &BigQuery{ Project: "test-project", Dataset: "test-dataset", - Timeout: testDuration, + Timeout: defaultTimeout, } - cerr := b.setUpTestClient() - require.NoError(t, cerr) - berr := b.Connect() - require.NoError(t, berr) + require.NoError(t, b.Init()) + require.NoError(t, b.setUpTestClient(srv.URL)) + require.NoError(t, b.Connect()) } func TestWrite(t *testing.T) { srv := localBigQueryServer(t) - testingHost = strings.ReplaceAll(srv.URL, "http://", "") defer srv.Close() b := &BigQuery{ Project: "test-project", Dataset: "test-dataset", - Timeout: testDuration, + Timeout: defaultTimeout, } mockMetrics := testutil.MockMetrics() - if err := b.setUpTestClient(); err != nil { - require.NoError(t, err) - } - if err := b.Connect(); err != nil { - require.NoError(t, err) - } + require.NoError(t, b.Init()) + require.NoError(t, b.setUpTestClient(srv.URL)) + require.NoError(t, b.Connect()) - if err := b.Write(mockMetrics); err != nil { - require.NoError(t, err) - } + require.NoError(t, b.Write(mockMetrics)) var rows []map[string]json.RawMessage - if err := json.Unmarshal(receivedBody["rows"], &rows); err != nil { - require.NoError(t, err) - } + require.NoError(t, json.Unmarshal(receivedBody["rows"], &rows)) var row Row - if err := json.Unmarshal(rows[0]["json"], &row); err != nil { - require.NoError(t, err) - } + require.NoError(t, json.Unmarshal(rows[0]["json"], &row)) pt, _ := time.Parse(time.RFC3339, row.Timestamp) require.Equal(t, mockMetrics[0].Tags()["tag1"], row.Tag1) @@ -87,49 +148,13 @@ func TestWrite(t *testing.T) { require.Equal(t, mockMetrics[0].Fields()["value"], row.Value) } -func TestMetricToTableDefault(t *testing.T) { - b := &BigQuery{ - Project: "test-project", - Dataset: "test-dataset", - Timeout: testDuration, - warnedOnHyphens: make(map[string]bool), - ReplaceHyphenTo: "_", - Log: testutil.Logger{}, - } - - otn := "table-with-hyphens" - ntn := b.metricToTable(otn) - - require.Equal(t, "table_with_hyphens", ntn) - require.True(t, b.warnedOnHyphens[otn]) -} - -func TestMetricToTableCustom(t *testing.T) { - log := testutil.Logger{} - - b := &BigQuery{ - Project: "test-project", - Dataset: "test-dataset", - Timeout: testDuration, - warnedOnHyphens: make(map[string]bool), - ReplaceHyphenTo: "*", - Log: log, - } - - otn := "table-with-hyphens" - ntn := b.metricToTable(otn) - - require.Equal(t, "table*with*hyphens", ntn) - require.True(t, b.warnedOnHyphens[otn]) -} - -func (b *BigQuery) setUpTestClient() error { +func (b *BigQuery) setUpTestClient(endpointURL string) error { noAuth := option.WithoutAuthentication() - endpoints := option.WithEndpoint("http://" + testingHost) + endpoint := option.WithEndpoint(endpointURL) ctx := context.Background() - c, err := bigquery.NewClient(ctx, b.Project, noAuth, endpoints) + c, err := bigquery.NewClient(ctx, b.Project, noAuth, endpoint) if err != nil { return err @@ -147,15 +172,11 @@ func localBigQueryServer(t *testing.T) *httptest.Server { switch r.URL.Path { case "/projects/test-project/datasets/test-dataset/tables/test1/insertAll": decoder := json.NewDecoder(r.Body) - - if err := decoder.Decode(&receivedBody); err != nil { - require.NoError(t, err) - } + require.NoError(t, decoder.Decode(&receivedBody)) w.WriteHeader(http.StatusOK) - if _, err := w.Write([]byte(successfulResponse)); err != nil { - require.NoError(t, err) - } + _, err := w.Write([]byte(successfulResponse)) + require.NoError(t, err) default: w.WriteHeader(http.StatusNotFound) } From 2bec266128c8a3efa9b918701ede340019253857 Mon Sep 17 00:00:00 2001 From: Thomas Casteleyn Date: Tue, 10 Oct 2023 17:40:44 +0200 Subject: [PATCH 4/5] fix: make check-deps --- docs/LICENSE_OF_DEPENDENCIES.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/LICENSE_OF_DEPENDENCIES.md b/docs/LICENSE_OF_DEPENDENCIES.md index 7ef77d162d830..048343c32e81c 100644 --- a/docs/LICENSE_OF_DEPENDENCIES.md +++ b/docs/LICENSE_OF_DEPENDENCIES.md @@ -377,7 +377,6 @@ following works: - golang.zx2c4.com/wireguard/wgctrl [MIT License](https://github.com/WireGuard/wgctrl-go/blob/master/LICENSE.md) - gonum.org/v1/gonum [BSD 3-Clause "New" or "Revised" License](https://github.com/gonum/gonum/blob/master/LICENSE) - google.golang.org/api [BSD 3-Clause "New" or "Revised" License](https://github.com/googleapis/google-api-go-client/blob/master/LICENSE) -- google.golang.org/appengine [Apache License 2.0](https://github.com/golang/appengine/blob/master/LICENSE) - google.golang.org/genproto [Apache License 2.0](https://github.com/google/go-genproto/blob/master/LICENSE) - google.golang.org/genproto/googleapis/api [Apache License 2.0](https://pkg.go.dev/google.golang.org/genproto/googleapis/api?tab=licenses) - google.golang.org/genproto/googleapis/rpc [Apache License 2.0](https://pkg.go.dev/google.golang.org/genproto/googleapis/rpc?tab=licenses) From ac3c8b9e90b254540249e4bd17f28a6221d4bb6d Mon Sep 17 00:00:00 2001 From: Thomas Casteleyn Date: Mon, 16 Oct 2023 09:50:43 +0200 Subject: [PATCH 5/5] feat(outputs.bigquery): Empty project means to auto-detect it --- plugins/outputs/bigquery/README.md | 5 ++++- plugins/outputs/bigquery/bigquery.go | 5 ++++- plugins/outputs/bigquery/sample.conf | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/plugins/outputs/bigquery/README.md b/plugins/outputs/bigquery/README.md index d65527161bc90..0274336516112 100644 --- a/plugins/outputs/bigquery/README.md +++ b/plugins/outputs/bigquery/README.md @@ -26,7 +26,7 @@ See the [CONFIGURATION.md][CONFIGURATION.md] for more details. credentials_file = "/path/to/service/account/key.json" ## Google Cloud Platform Project - # project = "*detect-project-id*" + # project = "" ## The namespace for the metric descriptor dataset = "telegraf" @@ -38,6 +38,9 @@ See the [CONFIGURATION.md][CONFIGURATION.md] for more details. # replace_hyphen_to = "_" ``` +Leaving `project` empty indicates the plugin will try to retrieve the project +from the credentials file. + Requires `dataset` to specify under which BigQuery dataset the corresponding metrics tables reside. diff --git a/plugins/outputs/bigquery/bigquery.go b/plugins/outputs/bigquery/bigquery.go index 6a0da84d3623a..6190ca44cd17d 100644 --- a/plugins/outputs/bigquery/bigquery.go +++ b/plugins/outputs/bigquery/bigquery.go @@ -47,6 +47,10 @@ func (*BigQuery) SampleConfig() string { } func (s *BigQuery) Init() error { + if s.Project == "" { + s.Project = bigquery.DetectProjectID + } + if s.Dataset == "" { return errors.New(`"dataset" is required`) } @@ -223,7 +227,6 @@ func (s *BigQuery) Close() error { func init() { outputs.Add("bigquery", func() telegraf.Output { return &BigQuery{ - Project: bigquery.DetectProjectID, Timeout: defaultTimeout, ReplaceHyphenTo: "_", } diff --git a/plugins/outputs/bigquery/sample.conf b/plugins/outputs/bigquery/sample.conf index 7e9384b14045b..b556195f92edd 100644 --- a/plugins/outputs/bigquery/sample.conf +++ b/plugins/outputs/bigquery/sample.conf @@ -4,7 +4,7 @@ credentials_file = "/path/to/service/account/key.json" ## Google Cloud Platform Project - # project = "*detect-project-id*" + # project = "" ## The namespace for the metric descriptor dataset = "telegraf"