From 8c916f8c3f4f31eb5b4a6592a40fba341251771d Mon Sep 17 00:00:00 2001 From: roypeter <16620459+roypeter@users.noreply.github.com> Date: Tue, 3 Sep 2024 16:11:01 +0530 Subject: [PATCH 1/9] [WIP] Use single metrics for dhis2 system --- dhis2/exporter.go | 54 +++++++++++++++++++++++------------------------ main.go | 8 +++++-- 2 files changed, 32 insertions(+), 30 deletions(-) diff --git a/dhis2/exporter.go b/dhis2/exporter.go index 319e325..b741985 100644 --- a/dhis2/exporter.go +++ b/dhis2/exporter.go @@ -2,37 +2,23 @@ package dhis2 import ( "log" - "strings" + "sync" "github.com/prometheus/client_golang/prometheus" ) -// sanitizeName prepares a valid prometheus metric name from a given URL -func sanitizeName(url string) string { - // Remove the protocol part - cleanURL := strings.TrimPrefix(url, "https://") - cleanURL = strings.TrimPrefix(cleanURL, "http://") - - // Replace dots and dashes with underscores - cleanURL = strings.ReplaceAll(cleanURL, "-", "_") - cleanURL = strings.ReplaceAll(cleanURL, ".", "_") - - return cleanURL -} - type Exporter struct { - client *Client - - info *prometheus.GaugeVec + clients []*Client + info *prometheus.GaugeVec + mutex sync.Mutex } -func NewExporter(client *Client) *Exporter { - dynamicName := "system_info_" + sanitizeName(client.BaseURL) +func NewExporter(clients []*Client) *Exporter { return &Exporter{ - client: client, + clients: clients, info: prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: "dhis2", - Name: dynamicName, + Name: "system_info", Help: "Information about the DHIS2 system", }, []string{"version", "revision", "contextPath", "buildTime"}), } @@ -46,14 +32,26 @@ func (e *Exporter) Describe(ch chan<- *prometheus.Desc) { // Collect is called by the Prometheus registry when collecting metrics. func (e *Exporter) Collect(ch chan<- prometheus.Metric) { - info, err := e.client.GetInfo() - if err != nil { - log.Printf("Failed to get dhis2 system information: %v\n", err) - return // Early return on error to avoid using uninitialized info - } + var wg sync.WaitGroup + + for _, client := range e.clients { + wg.Add(1) + + go func(client *Client) { + defer wg.Done() - // Set the version and revision as labels; gauge value is less meaningful here, just set to 1 - e.info.WithLabelValues(info.Version, info.Revision, info.ContextPath, info.BuildTime).Set(1) + info, err := client.GetInfo() + if err != nil { + log.Printf("Failed to get system information from %s: %v\n", client.BaseURL, err) + return + } + + e.mutex.Lock() + e.info.WithLabelValues(info.Version, info.Revision, info.ContextPath, info.BuildTime).Set(1) + e.mutex.Unlock() + }(client) + } + wg.Wait() e.info.Collect(ch) } diff --git a/main.go b/main.go index 67b51a5..5e87fc0 100644 --- a/main.go +++ b/main.go @@ -73,6 +73,7 @@ func main() { log.Fatalf("Error reading config file: %v", err) } + // Alphasms if config.ALPHASMSAPIKey == "" { log.Fatalf("ALPHASMS_API_KEY not provided in config file") } @@ -80,15 +81,18 @@ func main() { alphasmsExporter := alphasms.NewExporter(&alphasmsClient) prometheus.MustRegister(alphasmsExporter) + // DHIS2 + dhis2Clients := []*dhis2.Client{} for _, endpoint := range config.DHIS2Endpoints { dhis2Client := dhis2.Client{ Username: endpoint.Username, Password: endpoint.Password, BaseURL: endpoint.BaseURL, } - dhis2Exporter := dhis2.NewExporter(&dhis2Client) - prometheus.MustRegister(dhis2Exporter) + dhis2Clients = append(dhis2Clients, &dhis2Client) } + dhis2Exporter := dhis2.NewExporter(dhis2Clients) + prometheus.MustRegister(dhis2Exporter) // Register SendGrid exporters apiKeys := make(map[string]string) From d8f2e592a538d1c7571528c881284fb8e66e2107 Mon Sep 17 00:00:00 2001 From: roypeter <16620459+roypeter@users.noreply.github.com> Date: Wed, 4 Sep 2024 11:41:21 +0530 Subject: [PATCH 2/9] DHIS2 metrics validate http status code --- dhis2/client.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dhis2/client.go b/dhis2/client.go index 98d5d8f..acfbb8e 100644 --- a/dhis2/client.go +++ b/dhis2/client.go @@ -64,6 +64,11 @@ func (c *Client) doRequest(path string, result interface{}) error { } defer resp.Body.Close() + // Check if the status code is not 200 + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("unexpected status code: %d", resp.StatusCode) + } + // Read the body body, err := ioutil.ReadAll(resp.Body) if err != nil { From 0be386861efe8f605397a1274fd2e5b3c8a399ab Mon Sep 17 00:00:00 2001 From: roypeter <16620459+roypeter@users.noreply.github.com> Date: Wed, 4 Sep 2024 11:58:14 +0530 Subject: [PATCH 3/9] Use normal http timeout instead of context --- dhis2/client.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/dhis2/client.go b/dhis2/client.go index acfbb8e..9433221 100644 --- a/dhis2/client.go +++ b/dhis2/client.go @@ -1,7 +1,6 @@ package dhis2 import ( - "context" "encoding/base64" "encoding/json" "fmt" @@ -38,14 +37,10 @@ func (c *Client) GetInfo() (*Info, error) { // common method to do request func (c *Client) doRequest(path string, result interface{}) error { - // Create a context with a timeout - ctx, cancel := context.WithTimeout(context.Background(), httpTimeOutSec*time.Second) - defer cancel() - url := fmt.Sprintf("%s%s", c.BaseURL, path) // Create a new request - req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + req, err := http.NewRequest("GET", url, nil) if err != nil { return err } @@ -57,7 +52,9 @@ func (c *Client) doRequest(path string, result interface{}) error { req.Header.Set("Authorization", "Basic "+credentials) // Make the request - client := &http.Client{} + client := &http.Client{ + Timeout: httpTimeOutSec * time.Second, + } resp, err := client.Do(req) if err != nil { return err From 096b90800bae361e1460a66c12ead9c9d6bb5261 Mon Sep 17 00:00:00 2001 From: roypeter <16620459+roypeter@users.noreply.github.com> Date: Wed, 4 Sep 2024 12:35:06 +0530 Subject: [PATCH 4/9] Use connect timeout instead of total http timeout --- .gitignore | 1 + dhis2/client.go | 12 ++++++++++-- dhis2/exporter.go | 7 +++---- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 8f2ea6b..c566138 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ config.yaml .DS_Store +rtsl_exporter diff --git a/dhis2/client.go b/dhis2/client.go index 9433221..71f664c 100644 --- a/dhis2/client.go +++ b/dhis2/client.go @@ -5,11 +5,12 @@ import ( "encoding/json" "fmt" "io/ioutil" + "net" "net/http" "time" ) -const httpTimeOutSec = 1 +const httpConnectionSec = 1 type Client struct { Username string @@ -51,9 +52,16 @@ func (c *Client) doRequest(path string, result interface{}) error { // Set Authorization header req.Header.Set("Authorization", "Basic "+credentials) + // Custom Transport with Connect Timeout + transport := &http.Transport{ + DialContext: (&net.Dialer{ + Timeout: httpConnectionSec * time.Second, + }).DialContext, + } + // Make the request client := &http.Client{ - Timeout: httpTimeOutSec * time.Second, + Transport: transport, } resp, err := client.Do(req) if err != nil { diff --git a/dhis2/exporter.go b/dhis2/exporter.go index b741985..7db8bfc 100644 --- a/dhis2/exporter.go +++ b/dhis2/exporter.go @@ -33,22 +33,21 @@ func (e *Exporter) Describe(ch chan<- *prometheus.Desc) { // Collect is called by the Prometheus registry when collecting metrics. func (e *Exporter) Collect(ch chan<- prometheus.Metric) { var wg sync.WaitGroup + var mu sync.Mutex for _, client := range e.clients { wg.Add(1) - go func(client *Client) { defer wg.Done() - info, err := client.GetInfo() if err != nil { log.Printf("Failed to get system information from %s: %v\n", client.BaseURL, err) return } - e.mutex.Lock() + mu.Lock() e.info.WithLabelValues(info.Version, info.Revision, info.ContextPath, info.BuildTime).Set(1) - e.mutex.Unlock() + mu.Unlock() }(client) } From adcbd7a54362e56277ea56998a90936ab04b4a30 Mon Sep 17 00:00:00 2001 From: roypeter <16620459+roypeter@users.noreply.github.com> Date: Wed, 4 Sep 2024 12:54:52 +0530 Subject: [PATCH 5/9] Publish DHIS2 metrics on endpoint timeout as well --- dhis2/exporter.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dhis2/exporter.go b/dhis2/exporter.go index 7db8bfc..f389499 100644 --- a/dhis2/exporter.go +++ b/dhis2/exporter.go @@ -42,11 +42,14 @@ func (e *Exporter) Collect(ch chan<- prometheus.Metric) { info, err := client.GetInfo() if err != nil { log.Printf("Failed to get system information from %s: %v\n", client.BaseURL, err) + mu.Lock() + e.info.WithLabelValues("", "", client.BaseURL, "").Set(0) + mu.Unlock() return } mu.Lock() - e.info.WithLabelValues(info.Version, info.Revision, info.ContextPath, info.BuildTime).Set(1) + e.info.WithLabelValues(info.Version, info.Revision, client.BaseURL, info.BuildTime).Set(1) mu.Unlock() }(client) } From 2b65275282e1881fda93f6f2ac4680af5193f5d9 Mon Sep 17 00:00:00 2001 From: roypeter <16620459+roypeter@users.noreply.github.com> Date: Wed, 4 Sep 2024 13:11:34 +0530 Subject: [PATCH 6/9] Remove unused mutex in client --- dhis2/exporter.go | 1 - 1 file changed, 1 deletion(-) diff --git a/dhis2/exporter.go b/dhis2/exporter.go index f389499..50fd1a9 100644 --- a/dhis2/exporter.go +++ b/dhis2/exporter.go @@ -10,7 +10,6 @@ import ( type Exporter struct { clients []*Client info *prometheus.GaugeVec - mutex sync.Mutex } func NewExporter(clients []*Client) *Exporter { From 7f61db7c82679b7fc1cccffd00eb12c7eef32630 Mon Sep 17 00:00:00 2001 From: roypeter <16620459+roypeter@users.noreply.github.com> Date: Mon, 9 Sep 2024 11:22:27 +0530 Subject: [PATCH 7/9] Improve error logging --- dhis2/exporter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhis2/exporter.go b/dhis2/exporter.go index 50fd1a9..1935fff 100644 --- a/dhis2/exporter.go +++ b/dhis2/exporter.go @@ -40,7 +40,7 @@ func (e *Exporter) Collect(ch chan<- prometheus.Metric) { defer wg.Done() info, err := client.GetInfo() if err != nil { - log.Printf("Failed to get system information from %s: %v\n", client.BaseURL, err) + log.Printf("ERROR: Failed to get system information from %s: %v\n", client.BaseURL, err) mu.Lock() e.info.WithLabelValues("", "", client.BaseURL, "").Set(0) mu.Unlock() From 2dca65b543a1a445d2b65c5c980280fe049ed58e Mon Sep 17 00:00:00 2001 From: roypeter <16620459+roypeter@users.noreply.github.com> Date: Mon, 9 Sep 2024 11:46:57 +0530 Subject: [PATCH 8/9] Improve http client error handling and config --- dhis2/client.go | 26 +++++++++++++++----------- main.go | 12 +++++++----- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/dhis2/client.go b/dhis2/client.go index 71f664c..72da011 100644 --- a/dhis2/client.go +++ b/dhis2/client.go @@ -4,18 +4,19 @@ import ( "encoding/base64" "encoding/json" "fmt" - "io/ioutil" + "io" "net" "net/http" "time" ) -const httpConnectionSec = 1 +const DefaultConnectionTimeout = 1 * time.Second type Client struct { - Username string - Password string - BaseURL string + Username string + Password string + BaseURL string + ConnectionTimeout time.Duration } type Info struct { @@ -43,7 +44,7 @@ func (c *Client) doRequest(path string, result interface{}) error { // Create a new request req, err := http.NewRequest("GET", url, nil) if err != nil { - return err + return fmt.Errorf("failed to create request: %w", err) } // Encode credentials @@ -55,7 +56,7 @@ func (c *Client) doRequest(path string, result interface{}) error { // Custom Transport with Connect Timeout transport := &http.Transport{ DialContext: (&net.Dialer{ - Timeout: httpConnectionSec * time.Second, + Timeout: c.ConnectionTimeout, }).DialContext, } @@ -65,7 +66,7 @@ func (c *Client) doRequest(path string, result interface{}) error { } resp, err := client.Do(req) if err != nil { - return err + return fmt.Errorf("request failed: %w", err) } defer resp.Body.Close() @@ -75,12 +76,15 @@ func (c *Client) doRequest(path string, result interface{}) error { } // Read the body - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { - return err + return fmt.Errorf("failed to read response body: %w", err) } // Unmarshal JSON response into result err = json.Unmarshal(body, result) - return err + if err != nil { + return fmt.Errorf("failed to unmarshal response: %w", err) + } + return nil } diff --git a/main.go b/main.go index 5e87fc0..bd10358 100644 --- a/main.go +++ b/main.go @@ -2,12 +2,13 @@ package main import ( "context" + "io/ioutil" "log" "net/http" "os" "os/signal" - "io/ioutil" "time" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/simpledotorg/rtsl_exporter/alphasms" @@ -17,7 +18,7 @@ import ( ) type Config struct { - ALPHASMSAPIKey string `yaml:"alphasms_api_key"` + ALPHASMSAPIKey string `yaml:"alphasms_api_key"` SendGridAccounts []struct { AccountName string `yaml:"account_name"` APIKey string `yaml:"api_key"` @@ -85,9 +86,10 @@ func main() { dhis2Clients := []*dhis2.Client{} for _, endpoint := range config.DHIS2Endpoints { dhis2Client := dhis2.Client{ - Username: endpoint.Username, - Password: endpoint.Password, - BaseURL: endpoint.BaseURL, + Username: endpoint.Username, + Password: endpoint.Password, + BaseURL: endpoint.BaseURL, + ConnectionTimeout: dhis2.DefaultConnectionTimeout, } dhis2Clients = append(dhis2Clients, &dhis2Client) } From 0be2841c20c5f1cee07f4645c6f6926b79c52cd2 Mon Sep 17 00:00:00 2001 From: roypeter <16620459+roypeter@users.noreply.github.com> Date: Mon, 9 Sep 2024 11:50:38 +0530 Subject: [PATCH 9/9] Remove metrics update on failure --- dhis2/exporter.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/dhis2/exporter.go b/dhis2/exporter.go index 1935fff..acf8e10 100644 --- a/dhis2/exporter.go +++ b/dhis2/exporter.go @@ -41,9 +41,6 @@ func (e *Exporter) Collect(ch chan<- prometheus.Metric) { info, err := client.GetInfo() if err != nil { log.Printf("ERROR: Failed to get system information from %s: %v\n", client.BaseURL, err) - mu.Lock() - e.info.WithLabelValues("", "", client.BaseURL, "").Set(0) - mu.Unlock() return }