-
Notifications
You must be signed in to change notification settings - Fork 316
/
http_config.go
1313 lines (1163 loc) · 39.4 KB
/
http_config.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// Copyright 2016 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package config
import (
"bytes"
"context"
"crypto/sha256"
"crypto/tls"
"crypto/x509"
"encoding/json"
"fmt"
"net"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"sync"
"time"
conntrack "github.com/mwitkow/go-conntrack"
"golang.org/x/net/http/httpproxy"
"golang.org/x/net/http2"
"golang.org/x/oauth2"
"golang.org/x/oauth2/clientcredentials"
"gopkg.in/yaml.v2"
)
var (
// DefaultHTTPClientConfig is the default HTTP client configuration.
DefaultHTTPClientConfig = HTTPClientConfig{
FollowRedirects: true,
EnableHTTP2: true,
}
// defaultHTTPClientOptions holds the default HTTP client options.
defaultHTTPClientOptions = httpClientOptions{
keepAlivesEnabled: true,
http2Enabled: true,
// 5 minutes is typically above the maximum sane scrape interval. So we can
// use keepalive for all configurations.
idleConnTimeout: 5 * time.Minute,
}
)
type closeIdler interface {
CloseIdleConnections()
}
type TLSVersion uint16
var TLSVersions = map[string]TLSVersion{
"TLS13": (TLSVersion)(tls.VersionTLS13),
"TLS12": (TLSVersion)(tls.VersionTLS12),
"TLS11": (TLSVersion)(tls.VersionTLS11),
"TLS10": (TLSVersion)(tls.VersionTLS10),
}
func (tv *TLSVersion) UnmarshalYAML(unmarshal func(interface{}) error) error {
var s string
err := unmarshal((*string)(&s))
if err != nil {
return err
}
if v, ok := TLSVersions[s]; ok {
*tv = v
return nil
}
return fmt.Errorf("unknown TLS version: %s", s)
}
func (tv TLSVersion) MarshalYAML() (interface{}, error) {
for s, v := range TLSVersions {
if tv == v {
return s, nil
}
}
return nil, fmt.Errorf("unknown TLS version: %d", tv)
}
// MarshalJSON implements the json.Unmarshaler interface for TLSVersion.
func (tv *TLSVersion) UnmarshalJSON(data []byte) error {
var s string
if err := json.Unmarshal(data, &s); err != nil {
return err
}
if v, ok := TLSVersions[s]; ok {
*tv = v
return nil
}
return fmt.Errorf("unknown TLS version: %s", s)
}
// MarshalJSON implements the json.Marshaler interface for TLSVersion.
func (tv TLSVersion) MarshalJSON() ([]byte, error) {
for s, v := range TLSVersions {
if tv == v {
return json.Marshal(s)
}
}
return nil, fmt.Errorf("unknown TLS version: %d", tv)
}
// String implements the fmt.Stringer interface for TLSVersion.
func (tv *TLSVersion) String() string {
if tv == nil || *tv == 0 {
return ""
}
for s, v := range TLSVersions {
if *tv == v {
return s
}
}
return fmt.Sprintf("%d", tv)
}
// BasicAuth contains basic HTTP authentication credentials.
type BasicAuth struct {
Username string `yaml:"username" json:"username"`
UsernameFile string `yaml:"username_file,omitempty" json:"username_file,omitempty"`
Password Secret `yaml:"password,omitempty" json:"password,omitempty"`
PasswordFile string `yaml:"password_file,omitempty" json:"password_file,omitempty"`
}
// SetDirectory joins any relative file paths with dir.
func (a *BasicAuth) SetDirectory(dir string) {
if a == nil {
return
}
a.PasswordFile = JoinDir(dir, a.PasswordFile)
a.UsernameFile = JoinDir(dir, a.UsernameFile)
}
// Authorization contains HTTP authorization credentials.
type Authorization struct {
Type string `yaml:"type,omitempty" json:"type,omitempty"`
Credentials Secret `yaml:"credentials,omitempty" json:"credentials,omitempty"`
CredentialsFile string `yaml:"credentials_file,omitempty" json:"credentials_file,omitempty"`
}
// SetDirectory joins any relative file paths with dir.
func (a *Authorization) SetDirectory(dir string) {
if a == nil {
return
}
a.CredentialsFile = JoinDir(dir, a.CredentialsFile)
}
// URL is a custom URL type that allows validation at configuration load time.
type URL struct {
*url.URL
}
// UnmarshalYAML implements the yaml.Unmarshaler interface for URLs.
func (u *URL) UnmarshalYAML(unmarshal func(interface{}) error) error {
var s string
if err := unmarshal(&s); err != nil {
return err
}
urlp, err := url.Parse(s)
if err != nil {
return err
}
u.URL = urlp
return nil
}
// MarshalYAML implements the yaml.Marshaler interface for URLs.
func (u URL) MarshalYAML() (interface{}, error) {
if u.URL != nil {
return u.Redacted(), nil
}
return nil, nil
}
// Redacted returns the URL but replaces any password with "xxxxx".
func (u URL) Redacted() string {
if u.URL == nil {
return ""
}
ru := *u.URL
if _, ok := ru.User.Password(); ok {
// We can not use secretToken because it would be escaped.
ru.User = url.UserPassword(ru.User.Username(), "xxxxx")
}
return ru.String()
}
// UnmarshalJSON implements the json.Marshaler interface for URL.
func (u *URL) UnmarshalJSON(data []byte) error {
var s string
if err := json.Unmarshal(data, &s); err != nil {
return err
}
urlp, err := url.Parse(s)
if err != nil {
return err
}
u.URL = urlp
return nil
}
// MarshalJSON implements the json.Marshaler interface for URL.
func (u URL) MarshalJSON() ([]byte, error) {
if u.URL != nil {
return json.Marshal(u.URL.String())
}
return []byte("null"), nil
}
// OAuth2 is the oauth2 client configuration.
type OAuth2 struct {
ClientID string `yaml:"client_id" json:"client_id"`
ClientSecret Secret `yaml:"client_secret" json:"client_secret"`
ClientSecretFile string `yaml:"client_secret_file" json:"client_secret_file"`
Scopes []string `yaml:"scopes,omitempty" json:"scopes,omitempty"`
TokenURL string `yaml:"token_url" json:"token_url"`
EndpointParams map[string]string `yaml:"endpoint_params,omitempty" json:"endpoint_params,omitempty"`
TLSConfig TLSConfig `yaml:"tls_config,omitempty"`
ProxyConfig `yaml:",inline"`
}
// UnmarshalYAML implements the yaml.Unmarshaler interface
func (o *OAuth2) UnmarshalYAML(unmarshal func(interface{}) error) error {
type plain OAuth2
if err := unmarshal((*plain)(o)); err != nil {
return err
}
return o.ProxyConfig.Validate()
}
// UnmarshalJSON implements the json.Marshaler interface for URL.
func (o *OAuth2) UnmarshalJSON(data []byte) error {
type plain OAuth2
if err := json.Unmarshal(data, (*plain)(o)); err != nil {
return err
}
return o.ProxyConfig.Validate()
}
// SetDirectory joins any relative file paths with dir.
func (a *OAuth2) SetDirectory(dir string) {
if a == nil {
return
}
a.ClientSecretFile = JoinDir(dir, a.ClientSecretFile)
a.TLSConfig.SetDirectory(dir)
}
// LoadHTTPConfig parses the YAML input s into a HTTPClientConfig.
func LoadHTTPConfig(s string) (*HTTPClientConfig, error) {
cfg := &HTTPClientConfig{}
err := yaml.UnmarshalStrict([]byte(s), cfg)
if err != nil {
return nil, err
}
return cfg, nil
}
// LoadHTTPConfigFile parses the given YAML file into a HTTPClientConfig.
func LoadHTTPConfigFile(filename string) (*HTTPClientConfig, []byte, error) {
content, err := os.ReadFile(filename)
if err != nil {
return nil, nil, err
}
cfg, err := LoadHTTPConfig(string(content))
if err != nil {
return nil, nil, err
}
cfg.SetDirectory(filepath.Dir(filepath.Dir(filename)))
return cfg, content, nil
}
// HTTPClientConfig configures an HTTP client.
type HTTPClientConfig struct {
// The HTTP basic authentication credentials for the targets.
BasicAuth *BasicAuth `yaml:"basic_auth,omitempty" json:"basic_auth,omitempty"`
// The HTTP authorization credentials for the targets.
Authorization *Authorization `yaml:"authorization,omitempty" json:"authorization,omitempty"`
// The OAuth2 client credentials used to fetch a token for the targets.
OAuth2 *OAuth2 `yaml:"oauth2,omitempty" json:"oauth2,omitempty"`
// The bearer token for the targets. Deprecated in favour of
// Authorization.Credentials.
BearerToken Secret `yaml:"bearer_token,omitempty" json:"bearer_token,omitempty"`
// The bearer token file for the targets. Deprecated in favour of
// Authorization.CredentialsFile.
BearerTokenFile string `yaml:"bearer_token_file,omitempty" json:"bearer_token_file,omitempty"`
// TLSConfig to use to connect to the targets.
TLSConfig TLSConfig `yaml:"tls_config,omitempty" json:"tls_config,omitempty"`
// FollowRedirects specifies whether the client should follow HTTP 3xx redirects.
// The omitempty flag is not set, because it would be hidden from the
// marshalled configuration when set to false.
FollowRedirects bool `yaml:"follow_redirects" json:"follow_redirects"`
// EnableHTTP2 specifies whether the client should configure HTTP2.
// The omitempty flag is not set, because it would be hidden from the
// marshalled configuration when set to false.
EnableHTTP2 bool `yaml:"enable_http2" json:"enable_http2"`
// Proxy configuration.
ProxyConfig `yaml:",inline"`
// HTTPHeaders specify headers to inject in the requests. Those headers
// could be marshalled back to the users.
HTTPHeaders *Headers `yaml:"http_headers" json:"http_headers"`
}
// SetDirectory joins any relative file paths with dir.
func (c *HTTPClientConfig) SetDirectory(dir string) {
if c == nil {
return
}
c.TLSConfig.SetDirectory(dir)
c.BasicAuth.SetDirectory(dir)
c.Authorization.SetDirectory(dir)
c.OAuth2.SetDirectory(dir)
c.HTTPHeaders.SetDirectory(dir)
c.BearerTokenFile = JoinDir(dir, c.BearerTokenFile)
}
// Validate validates the HTTPClientConfig to check only one of BearerToken,
// BasicAuth and BearerTokenFile is configured. It also validates that ProxyURL
// is set if ProxyConnectHeader is set.
func (c *HTTPClientConfig) Validate() error {
// Backwards compatibility with the bearer_token field.
if len(c.BearerToken) > 0 && len(c.BearerTokenFile) > 0 {
return fmt.Errorf("at most one of bearer_token & bearer_token_file must be configured")
}
if (c.BasicAuth != nil || c.OAuth2 != nil) && (len(c.BearerToken) > 0 || len(c.BearerTokenFile) > 0) {
return fmt.Errorf("at most one of basic_auth, oauth2, bearer_token & bearer_token_file must be configured")
}
if c.BasicAuth != nil && (string(c.BasicAuth.Username) != "" && c.BasicAuth.UsernameFile != "") {
return fmt.Errorf("at most one of basic_auth username & username_file must be configured")
}
if c.BasicAuth != nil && (string(c.BasicAuth.Password) != "" && c.BasicAuth.PasswordFile != "") {
return fmt.Errorf("at most one of basic_auth password & password_file must be configured")
}
if c.Authorization != nil {
if len(c.BearerToken) > 0 || len(c.BearerTokenFile) > 0 {
return fmt.Errorf("authorization is not compatible with bearer_token & bearer_token_file")
}
if string(c.Authorization.Credentials) != "" && c.Authorization.CredentialsFile != "" {
return fmt.Errorf("at most one of authorization credentials & credentials_file must be configured")
}
c.Authorization.Type = strings.TrimSpace(c.Authorization.Type)
if len(c.Authorization.Type) == 0 {
c.Authorization.Type = "Bearer"
}
if strings.ToLower(c.Authorization.Type) == "basic" {
return fmt.Errorf(`authorization type cannot be set to "basic", use "basic_auth" instead`)
}
if c.BasicAuth != nil || c.OAuth2 != nil {
return fmt.Errorf("at most one of basic_auth, oauth2 & authorization must be configured")
}
} else {
if len(c.BearerToken) > 0 {
c.Authorization = &Authorization{Credentials: c.BearerToken}
c.Authorization.Type = "Bearer"
c.BearerToken = ""
}
if len(c.BearerTokenFile) > 0 {
c.Authorization = &Authorization{CredentialsFile: c.BearerTokenFile}
c.Authorization.Type = "Bearer"
c.BearerTokenFile = ""
}
}
if c.OAuth2 != nil {
if c.BasicAuth != nil {
return fmt.Errorf("at most one of basic_auth, oauth2 & authorization must be configured")
}
if len(c.OAuth2.ClientID) == 0 {
return fmt.Errorf("oauth2 client_id must be configured")
}
if len(c.OAuth2.TokenURL) == 0 {
return fmt.Errorf("oauth2 token_url must be configured")
}
if len(c.OAuth2.ClientSecret) > 0 && len(c.OAuth2.ClientSecretFile) > 0 {
return fmt.Errorf("at most one of oauth2 client_secret & client_secret_file must be configured")
}
}
if err := c.ProxyConfig.Validate(); err != nil {
return err
}
if c.HTTPHeaders != nil {
if err := c.HTTPHeaders.Validate(); err != nil {
return err
}
}
return nil
}
// UnmarshalYAML implements the yaml.Unmarshaler interface
func (c *HTTPClientConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
type plain HTTPClientConfig
*c = DefaultHTTPClientConfig
if err := unmarshal((*plain)(c)); err != nil {
return err
}
return c.Validate()
}
// UnmarshalJSON implements the json.Marshaler interface for URL.
func (c *HTTPClientConfig) UnmarshalJSON(data []byte) error {
type plain HTTPClientConfig
*c = DefaultHTTPClientConfig
if err := json.Unmarshal(data, (*plain)(c)); err != nil {
return err
}
return c.Validate()
}
// UnmarshalYAML implements the yaml.Unmarshaler interface.
func (a *BasicAuth) UnmarshalYAML(unmarshal func(interface{}) error) error {
type plain BasicAuth
return unmarshal((*plain)(a))
}
// DialContextFunc defines the signature of the DialContext() function implemented
// by net.Dialer.
type DialContextFunc func(context.Context, string, string) (net.Conn, error)
type httpClientOptions struct {
dialContextFunc DialContextFunc
keepAlivesEnabled bool
http2Enabled bool
idleConnTimeout time.Duration
userAgent string
host string
}
// HTTPClientOption defines an option that can be applied to the HTTP client.
type HTTPClientOption func(options *httpClientOptions)
// WithDialContextFunc allows you to override func gets used for the actual dialing. The default is `net.Dialer.DialContext`.
func WithDialContextFunc(fn DialContextFunc) HTTPClientOption {
return func(opts *httpClientOptions) {
opts.dialContextFunc = fn
}
}
// WithKeepAlivesDisabled allows to disable HTTP keepalive.
func WithKeepAlivesDisabled() HTTPClientOption {
return func(opts *httpClientOptions) {
opts.keepAlivesEnabled = false
}
}
// WithHTTP2Disabled allows to disable HTTP2.
func WithHTTP2Disabled() HTTPClientOption {
return func(opts *httpClientOptions) {
opts.http2Enabled = false
}
}
// WithIdleConnTimeout allows setting the idle connection timeout.
func WithIdleConnTimeout(timeout time.Duration) HTTPClientOption {
return func(opts *httpClientOptions) {
opts.idleConnTimeout = timeout
}
}
// WithUserAgent allows setting the user agent.
func WithUserAgent(ua string) HTTPClientOption {
return func(opts *httpClientOptions) {
opts.userAgent = ua
}
}
// WithHost allows setting the host header.
func WithHost(host string) HTTPClientOption {
return func(opts *httpClientOptions) {
opts.host = host
}
}
// NewClient returns a http.Client using the specified http.RoundTripper.
func newClient(rt http.RoundTripper) *http.Client {
return &http.Client{Transport: rt}
}
// NewClientFromConfig returns a new HTTP client configured for the
// given config.HTTPClientConfig and config.HTTPClientOption.
// The name is used as go-conntrack metric label.
func NewClientFromConfig(cfg HTTPClientConfig, name string, optFuncs ...HTTPClientOption) (*http.Client, error) {
rt, err := NewRoundTripperFromConfig(cfg, name, optFuncs...)
if err != nil {
return nil, err
}
client := newClient(rt)
if !cfg.FollowRedirects {
client.CheckRedirect = func(*http.Request, []*http.Request) error {
return http.ErrUseLastResponse
}
}
return client, nil
}
// NewRoundTripperFromConfig returns a new HTTP RoundTripper configured for the
// given config.HTTPClientConfig and config.HTTPClientOption.
// The name is used as go-conntrack metric label.
func NewRoundTripperFromConfig(cfg HTTPClientConfig, name string, optFuncs ...HTTPClientOption) (http.RoundTripper, error) {
opts := defaultHTTPClientOptions
for _, f := range optFuncs {
f(&opts)
}
var dialContext func(ctx context.Context, network, addr string) (net.Conn, error)
if opts.dialContextFunc != nil {
dialContext = conntrack.NewDialContextFunc(
conntrack.DialWithDialContextFunc((func(context.Context, string, string) (net.Conn, error))(opts.dialContextFunc)),
conntrack.DialWithTracing(),
conntrack.DialWithName(name))
} else {
dialContext = conntrack.NewDialContextFunc(
conntrack.DialWithTracing(),
conntrack.DialWithName(name))
}
newRT := func(tlsConfig *tls.Config) (http.RoundTripper, error) {
// The only timeout we care about is the configured scrape timeout.
// It is applied on request. So we leave out any timings here.
var rt http.RoundTripper = &http.Transport{
Proxy: cfg.ProxyConfig.Proxy(),
ProxyConnectHeader: cfg.ProxyConfig.GetProxyConnectHeader(),
MaxIdleConns: 20000,
MaxIdleConnsPerHost: 1000, // see https://github.com/golang/go/issues/13801
DisableKeepAlives: !opts.keepAlivesEnabled,
TLSClientConfig: tlsConfig,
DisableCompression: true,
IdleConnTimeout: opts.idleConnTimeout,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
DialContext: dialContext,
}
if opts.http2Enabled && cfg.EnableHTTP2 {
// HTTP/2 support is golang had many problematic cornercases where
// dead connections would be kept and used in connection pools.
// https://github.com/golang/go/issues/32388
// https://github.com/golang/go/issues/39337
// https://github.com/golang/go/issues/39750
http2t, err := http2.ConfigureTransports(rt.(*http.Transport))
if err != nil {
return nil, err
}
http2t.ReadIdleTimeout = time.Minute
}
// If a authorization_credentials is provided, create a round tripper that will set the
// Authorization header correctly on each request.
if cfg.Authorization != nil && len(cfg.Authorization.CredentialsFile) > 0 {
rt = NewAuthorizationCredentialsFileRoundTripper(cfg.Authorization.Type, cfg.Authorization.CredentialsFile, rt)
} else if cfg.Authorization != nil {
rt = NewAuthorizationCredentialsRoundTripper(cfg.Authorization.Type, cfg.Authorization.Credentials, rt)
}
// Backwards compatibility, be nice with importers who would not have
// called Validate().
if len(cfg.BearerToken) > 0 {
rt = NewAuthorizationCredentialsRoundTripper("Bearer", cfg.BearerToken, rt)
} else if len(cfg.BearerTokenFile) > 0 {
rt = NewAuthorizationCredentialsFileRoundTripper("Bearer", cfg.BearerTokenFile, rt)
}
if cfg.BasicAuth != nil {
rt = NewBasicAuthRoundTripper(cfg.BasicAuth.Username, cfg.BasicAuth.Password, cfg.BasicAuth.UsernameFile, cfg.BasicAuth.PasswordFile, rt)
}
if cfg.OAuth2 != nil {
rt = NewOAuth2RoundTripper(cfg.OAuth2, rt, &opts)
}
if cfg.HTTPHeaders != nil {
rt = NewHeadersRoundTripper(cfg.HTTPHeaders, rt)
}
if opts.userAgent != "" {
rt = NewUserAgentRoundTripper(opts.userAgent, rt)
}
if opts.host != "" {
rt = NewHostRoundTripper(opts.host, rt)
}
// Return a new configured RoundTripper.
return rt, nil
}
tlsConfig, err := NewTLSConfig(&cfg.TLSConfig)
if err != nil {
return nil, err
}
if len(cfg.TLSConfig.CAFile) == 0 {
// No need for a RoundTripper that reloads the CA file automatically.
return newRT(tlsConfig)
}
return NewTLSRoundTripper(tlsConfig, cfg.TLSConfig.roundTripperSettings(), newRT)
}
type authorizationCredentialsRoundTripper struct {
authType string
authCredentials Secret
rt http.RoundTripper
}
// NewAuthorizationCredentialsRoundTripper adds the provided credentials to a
// request unless the authorization header has already been set.
func NewAuthorizationCredentialsRoundTripper(authType string, authCredentials Secret, rt http.RoundTripper) http.RoundTripper {
return &authorizationCredentialsRoundTripper{authType, authCredentials, rt}
}
func (rt *authorizationCredentialsRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
if len(req.Header.Get("Authorization")) == 0 {
req = cloneRequest(req)
req.Header.Set("Authorization", fmt.Sprintf("%s %s", rt.authType, string(rt.authCredentials)))
}
return rt.rt.RoundTrip(req)
}
func (rt *authorizationCredentialsRoundTripper) CloseIdleConnections() {
if ci, ok := rt.rt.(closeIdler); ok {
ci.CloseIdleConnections()
}
}
type authorizationCredentialsFileRoundTripper struct {
authType string
authCredentialsFile string
rt http.RoundTripper
}
// NewAuthorizationCredentialsFileRoundTripper adds the authorization
// credentials read from the provided file to a request unless the authorization
// header has already been set. This file is read for every request.
func NewAuthorizationCredentialsFileRoundTripper(authType, authCredentialsFile string, rt http.RoundTripper) http.RoundTripper {
return &authorizationCredentialsFileRoundTripper{authType, authCredentialsFile, rt}
}
func (rt *authorizationCredentialsFileRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
if len(req.Header.Get("Authorization")) == 0 {
b, err := os.ReadFile(rt.authCredentialsFile)
if err != nil {
return nil, fmt.Errorf("unable to read authorization credentials file %s: %w", rt.authCredentialsFile, err)
}
authCredentials := strings.TrimSpace(string(b))
req = cloneRequest(req)
req.Header.Set("Authorization", fmt.Sprintf("%s %s", rt.authType, authCredentials))
}
return rt.rt.RoundTrip(req)
}
func (rt *authorizationCredentialsFileRoundTripper) CloseIdleConnections() {
if ci, ok := rt.rt.(closeIdler); ok {
ci.CloseIdleConnections()
}
}
type basicAuthRoundTripper struct {
username string
password Secret
usernameFile string
passwordFile string
rt http.RoundTripper
}
// NewBasicAuthRoundTripper will apply a BASIC auth authorization header to a request unless it has
// already been set.
func NewBasicAuthRoundTripper(username string, password Secret, usernameFile, passwordFile string, rt http.RoundTripper) http.RoundTripper {
return &basicAuthRoundTripper{username, password, usernameFile, passwordFile, rt}
}
func (rt *basicAuthRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
var username string
var password string
if len(req.Header.Get("Authorization")) != 0 {
return rt.rt.RoundTrip(req)
}
if rt.usernameFile != "" {
usernameBytes, err := os.ReadFile(rt.usernameFile)
if err != nil {
return nil, fmt.Errorf("unable to read basic auth username file %s: %w", rt.usernameFile, err)
}
username = strings.TrimSpace(string(usernameBytes))
} else {
username = rt.username
}
if rt.passwordFile != "" {
passwordBytes, err := os.ReadFile(rt.passwordFile)
if err != nil {
return nil, fmt.Errorf("unable to read basic auth password file %s: %w", rt.passwordFile, err)
}
password = strings.TrimSpace(string(passwordBytes))
} else {
password = string(rt.password)
}
req = cloneRequest(req)
req.SetBasicAuth(username, password)
return rt.rt.RoundTrip(req)
}
func (rt *basicAuthRoundTripper) CloseIdleConnections() {
if ci, ok := rt.rt.(closeIdler); ok {
ci.CloseIdleConnections()
}
}
type oauth2RoundTripper struct {
config *OAuth2
rt http.RoundTripper
next http.RoundTripper
secret string
mtx sync.RWMutex
opts *httpClientOptions
client *http.Client
}
func NewOAuth2RoundTripper(config *OAuth2, next http.RoundTripper, opts *httpClientOptions) http.RoundTripper {
return &oauth2RoundTripper{
config: config,
next: next,
opts: opts,
}
}
func (rt *oauth2RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
var (
secret string
changed bool
)
if rt.config.ClientSecretFile != "" {
data, err := os.ReadFile(rt.config.ClientSecretFile)
if err != nil {
return nil, fmt.Errorf("unable to read oauth2 client secret file %s: %w", rt.config.ClientSecretFile, err)
}
secret = strings.TrimSpace(string(data))
rt.mtx.RLock()
changed = secret != rt.secret
rt.mtx.RUnlock()
} else {
// Either an inline secret or nothing (use an empty string) was provided.
secret = string(rt.config.ClientSecret)
}
if changed || rt.rt == nil {
config := &clientcredentials.Config{
ClientID: rt.config.ClientID,
ClientSecret: secret,
Scopes: rt.config.Scopes,
TokenURL: rt.config.TokenURL,
EndpointParams: mapToValues(rt.config.EndpointParams),
}
tlsConfig, err := NewTLSConfig(&rt.config.TLSConfig)
if err != nil {
return nil, err
}
tlsTransport := func(tlsConfig *tls.Config) (http.RoundTripper, error) {
return &http.Transport{
TLSClientConfig: tlsConfig,
Proxy: rt.config.ProxyConfig.Proxy(),
ProxyConnectHeader: rt.config.ProxyConfig.GetProxyConnectHeader(),
DisableKeepAlives: !rt.opts.keepAlivesEnabled,
MaxIdleConns: 20,
MaxIdleConnsPerHost: 1, // see https://github.com/golang/go/issues/13801
IdleConnTimeout: 10 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
}, nil
}
var t http.RoundTripper
if len(rt.config.TLSConfig.CAFile) == 0 {
t, _ = tlsTransport(tlsConfig)
} else {
t, err = NewTLSRoundTripper(tlsConfig, rt.config.TLSConfig.roundTripperSettings(), tlsTransport)
if err != nil {
return nil, err
}
}
if ua := req.UserAgent(); ua != "" {
t = NewUserAgentRoundTripper(ua, t)
}
client := &http.Client{Transport: t}
ctx := context.WithValue(context.Background(), oauth2.HTTPClient, client)
tokenSource := config.TokenSource(ctx)
rt.mtx.Lock()
rt.secret = secret
rt.rt = &oauth2.Transport{
Base: rt.next,
Source: tokenSource,
}
if rt.client != nil {
rt.client.CloseIdleConnections()
}
rt.client = client
rt.mtx.Unlock()
}
rt.mtx.RLock()
currentRT := rt.rt
rt.mtx.RUnlock()
return currentRT.RoundTrip(req)
}
func (rt *oauth2RoundTripper) CloseIdleConnections() {
if rt.client != nil {
rt.client.CloseIdleConnections()
}
if ci, ok := rt.next.(closeIdler); ok {
ci.CloseIdleConnections()
}
}
func mapToValues(m map[string]string) url.Values {
v := url.Values{}
for name, value := range m {
v.Set(name, value)
}
return v
}
// cloneRequest returns a clone of the provided *http.Request.
// The clone is a shallow copy of the struct and its Header map.
func cloneRequest(r *http.Request) *http.Request {
// Shallow copy of the struct.
r2 := new(http.Request)
*r2 = *r
// Deep copy of the Header.
r2.Header = make(http.Header)
for k, s := range r.Header {
r2.Header[k] = s
}
return r2
}
// NewTLSConfig creates a new tls.Config from the given TLSConfig.
func NewTLSConfig(cfg *TLSConfig) (*tls.Config, error) {
if err := cfg.Validate(); err != nil {
return nil, err
}
tlsConfig := &tls.Config{
InsecureSkipVerify: cfg.InsecureSkipVerify,
MinVersion: uint16(cfg.MinVersion),
MaxVersion: uint16(cfg.MaxVersion),
}
if cfg.MaxVersion != 0 && cfg.MinVersion != 0 {
if cfg.MaxVersion < cfg.MinVersion {
return nil, fmt.Errorf("tls_config.max_version must be greater than or equal to tls_config.min_version if both are specified")
}
}
// If a CA cert is provided then let's read it in so we can validate the
// scrape target's certificate properly.
if len(cfg.CA) > 0 {
if !updateRootCA(tlsConfig, []byte(cfg.CA)) {
return nil, fmt.Errorf("unable to use inline CA cert")
}
} else if len(cfg.CAFile) > 0 {
b, err := readCAFile(cfg.CAFile)
if err != nil {
return nil, err
}
if !updateRootCA(tlsConfig, b) {
return nil, fmt.Errorf("unable to use specified CA cert %s", cfg.CAFile)
}
}
if len(cfg.ServerName) > 0 {
tlsConfig.ServerName = cfg.ServerName
}
// If a client cert & key is provided then configure TLS config accordingly.
if cfg.usingClientCert() && cfg.usingClientKey() {
// Verify that client cert and key are valid.
if _, err := cfg.getClientCertificate(nil); err != nil {
return nil, err
}
tlsConfig.GetClientCertificate = cfg.getClientCertificate
}
return tlsConfig, nil
}
// TLSConfig configures the options for TLS connections.
type TLSConfig struct {
// Text of the CA cert to use for the targets.
CA string `yaml:"ca,omitempty" json:"ca,omitempty"`
// Text of the client cert file for the targets.
Cert string `yaml:"cert,omitempty" json:"cert,omitempty"`
// Text of the client key file for the targets.
Key Secret `yaml:"key,omitempty" json:"key,omitempty"`
// The CA cert to use for the targets.
CAFile string `yaml:"ca_file,omitempty" json:"ca_file,omitempty"`
// The client cert file for the targets.
CertFile string `yaml:"cert_file,omitempty" json:"cert_file,omitempty"`
// The client key file for the targets.
KeyFile string `yaml:"key_file,omitempty" json:"key_file,omitempty"`
// Used to verify the hostname for the targets.
ServerName string `yaml:"server_name,omitempty" json:"server_name,omitempty"`
// Disable target certificate validation.
InsecureSkipVerify bool `yaml:"insecure_skip_verify" json:"insecure_skip_verify"`
// Minimum TLS version.
MinVersion TLSVersion `yaml:"min_version,omitempty" json:"min_version,omitempty"`
// Maximum TLS version.
MaxVersion TLSVersion `yaml:"max_version,omitempty" json:"max_version,omitempty"`
}
// SetDirectory joins any relative file paths with dir.
func (c *TLSConfig) SetDirectory(dir string) {
if c == nil {
return
}
c.CAFile = JoinDir(dir, c.CAFile)
c.CertFile = JoinDir(dir, c.CertFile)
c.KeyFile = JoinDir(dir, c.KeyFile)
}
// UnmarshalYAML implements the yaml.Unmarshaler interface.
func (c *TLSConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
type plain TLSConfig
if err := unmarshal((*plain)(c)); err != nil {
return err
}
return c.Validate()
}
// Validate validates the TLSConfig to check that only one of the inlined or
// file-based fields for the TLS CA, client certificate, and client key are
// used.
func (c *TLSConfig) Validate() error {
if len(c.CA) > 0 && len(c.CAFile) > 0 {
return fmt.Errorf("at most one of ca and ca_file must be configured")
}
if len(c.Cert) > 0 && len(c.CertFile) > 0 {
return fmt.Errorf("at most one of cert and cert_file must be configured")
}
if len(c.Key) > 0 && len(c.KeyFile) > 0 {
return fmt.Errorf("at most one of key and key_file must be configured")
}
if c.usingClientCert() && !c.usingClientKey() {
return fmt.Errorf("exactly one of key or key_file must be configured when a client certificate is configured")
} else if c.usingClientKey() && !c.usingClientCert() {
return fmt.Errorf("exactly one of cert or cert_file must be configured when a client key is configured")
}
return nil
}
func (c *TLSConfig) usingClientCert() bool {
return len(c.Cert) > 0 || len(c.CertFile) > 0
}
func (c *TLSConfig) usingClientKey() bool {
return len(c.Key) > 0 || len(c.KeyFile) > 0
}
func (c *TLSConfig) roundTripperSettings() TLSRoundTripperSettings {
return TLSRoundTripperSettings{
CA: c.CA,
CAFile: c.CAFile,
Cert: c.Cert,
CertFile: c.CertFile,
Key: string(c.Key),
KeyFile: c.KeyFile,
}
}
// getClientCertificate reads the pair of client cert and key from disk and returns a tls.Certificate.
func (c *TLSConfig) getClientCertificate(_ *tls.CertificateRequestInfo) (*tls.Certificate, error) {
var (
certData, keyData []byte
err error
)
if c.CertFile != "" {
certData, err = os.ReadFile(c.CertFile)
if err != nil {