From 35e9b5da9f3c066b203ad19ca470b7c73ea06d92 Mon Sep 17 00:00:00 2001 From: Josh Powers Date: Mon, 11 Dec 2023 11:51:13 -0700 Subject: [PATCH 1/2] fix(inputs.php-fpm): Parse JSON output fixes: #14421 --- plugins/inputs/phpfpm/README.md | 39 +++++ plugins/inputs/phpfpm/phpfpm.go | 113 ++++++++++++- plugins/inputs/phpfpm/phpfpm_test.go | 30 ++++ plugins/inputs/phpfpm/sample.conf | 6 + plugins/inputs/phpfpm/testdata/expected.out | 11 ++ plugins/inputs/phpfpm/testdata/phpfpm.json | 168 ++++++++++++++++++++ 6 files changed, 361 insertions(+), 6 deletions(-) create mode 100644 plugins/inputs/phpfpm/testdata/expected.out create mode 100644 plugins/inputs/phpfpm/testdata/phpfpm.json diff --git a/plugins/inputs/phpfpm/README.md b/plugins/inputs/phpfpm/README.md index 920cf9686f700..b2f129b4f3f9a 100644 --- a/plugins/inputs/phpfpm/README.md +++ b/plugins/inputs/phpfpm/README.md @@ -39,6 +39,12 @@ See the [CONFIGURATION.md][CONFIGURATION.md] for more details. ## urls = ["http://192.168.1.20/status", "/tmp/fpm.sock"] urls = ["http://localhost/status"] + ## Format of stats to parse, set to "status" or "json" + ## If the user configures the URL to return JSON (e.g. + ## http://localhost/status?json), set to JSON. Otherwise, will attempt to + ## parse line-by-line. The JSON mode will produce additonal metrics. + # format = "status" + ## Duration allowed to complete HTTP requests. # timeout = "5s" @@ -70,6 +76,23 @@ host, and socket path is accessible to telegraf user. - max_active_processes - max_children_reached - slow_requests +- phpfpm_process + - tags: + - pool + - request method + - request uri + - script + - url + - user + - fields: + - content length + - last request cpu + - last request memory + - request duration + - requests + - start time + - start since + - state ## Example Output @@ -78,3 +101,19 @@ phpfpm,pool=www accepted_conn=13i,active_processes=2i,idle_processes=1i,listen_q phpfpm,pool=www2 accepted_conn=12i,active_processes=1i,idle_processes=2i,listen_queue=0i,listen_queue_len=0i,max_active_processes=2i,max_children_reached=0i,max_listen_queue=0i,slow_requests=0i,total_processes=3i 1453011293083691422 phpfpm,pool=www3 accepted_conn=11i,active_processes=1i,idle_processes=2i,listen_queue=0i,listen_queue_len=0i,max_active_processes=2i,max_children_reached=0i,max_listen_queue=0i,slow_requests=0i,total_processes=3i 1453011293083691658 ``` + +With the JSON output, additional metrics around processes are generated: + +```text +phpfpm,pool=www,url=http://127.0.0.1:44637?full&json accepted_conn=3879i,active_processes=1i,idle_processes=9i,listen_queue=0i,listen_queue_len=0i,max_active_processes=3i,max_children_reached=0i,max_listen_queue=0i,slow_requests=0i,start_since=4901i,total_processes=10i +phpfpm_process,pool=www,request_method=GET,request_uri=/fpm-status?json&full,script=-,url=http://127.0.0.1:44637?full&json,user=- content_length=0i,last_request_cpu=0,last_request_memory=0,request_duration=159i,requests=386i,start_time=1702044927i,state="Running" +phpfpm_process,pool=www,request_method=GET,request_uri=/fpm-status,script=-,url=http://127.0.0.1:44637?full&json,user=- content_length=0i,last_request_cpu=0,last_request_memory=2097152,request_duration=174i,requests=390i,start_time=1702044927i,state="Idle" +phpfpm_process,pool=www,request_method=GET,request_uri=/index.php,script=script.php,url=http://127.0.0.1:44637?full&json,user=- content_length=0i,last_request_cpu=104.93,last_request_memory=2097152,request_duration=9530i,requests=389i,start_time=1702044927i,state="Idle" +phpfpm_process,pool=www,request_method=GET,request_uri=/ping,script=-,url=http://127.0.0.1:44637?full&json,user=- content_length=0i,last_request_cpu=0,last_request_memory=2097152,request_duration=127i,requests=399i,start_time=1702044927i,state="Idle" +phpfpm_process,pool=www,request_method=GET,request_uri=/index.php,script=script.php,url=http://127.0.0.1:44637?full&json,user=- content_length=0i,last_request_cpu=0,last_request_memory=2097152,request_duration=9713i,requests=382i,start_time=1702044927i,state="Idle" +phpfpm_process,pool=www,request_method=GET,request_uri=/ping,script=-,url=http://127.0.0.1:44637?full&json,user=- content_length=0i,last_request_cpu=0,last_request_memory=2097152,request_duration=133i,requests=383i,start_time=1702044927i,state="Idle" +phpfpm_process,pool=www,request_method=GET,request_uri=/fpm-status?json,script=-,url=http://127.0.0.1:44637?full&json,user=- content_length=0i,last_request_cpu=0,last_request_memory=2097152,request_duration=154i,requests=381i,start_time=1702044927i,state="Idle" +phpfpm_process,pool=www,request_method=GET,request_uri=/ping,script=-,url=http://127.0.0.1:44637?full&json,user=- content_length=0i,last_request_cpu=0,last_request_memory=2097152,request_duration=108i,requests=397i,start_time=1702044927i,state="Idle" +phpfpm_process,pool=www,request_method=GET,request_uri=/index.php,script=script.php,url=http://127.0.0.1:44637?full&json,user=- content_length=0i,last_request_cpu=110.28,last_request_memory=2097152,request_duration=9068i,requests=381i,start_time=1702044927i,state="Idle" +phpfpm_process,pool=www,request_method=GET,request_uri=/index.php,script=script.php,url=http://127.0.0.1:44637?full&json,user=- content_length=0i,last_request_cpu=64.27,last_request_memory=2097152,request_duration=15559i,requests=391i,start_time=1702044927i,state="Idle" +``` diff --git a/plugins/inputs/phpfpm/phpfpm.go b/plugins/inputs/phpfpm/phpfpm.go index 8b4efbeda525f..9521b936165cb 100644 --- a/plugins/inputs/phpfpm/phpfpm.go +++ b/plugins/inputs/phpfpm/phpfpm.go @@ -5,6 +5,7 @@ import ( "bufio" "bytes" _ "embed" + "encoding/json" "fmt" "io" "net/http" @@ -40,15 +41,49 @@ const ( PfSlowRequests = "slow requests" ) +type JSONMetrics struct { + Pool string `json:"pool"` + ProcessManager string `json:"process manager"` + StartTime int `json:"start time"` + StartSince int `json:"start since"` + AcceptedConn int `json:"accepted conn"` + ListenQueue int `json:"listen queue"` + MaxListenQueue int `json:"max listen queue"` + ListenQueueLen int `json:"listen queue len"` + IdleProcesses int `json:"idle processes"` + ActiveProcesses int `json:"active processes"` + TotalProcesses int `json:"total processes"` + MaxActiveProcesses int `json:"max active processes"` + MaxChildrenReached int `json:"max children reached"` + SlowRequests int `json:"slow requests"` + Processes []struct { + Pid int `json:"pid"` + State string `json:"state"` + StartTime int `json:"start time"` + StartSince int `json:"start since"` + Requests int `json:"requests"` + RequestDuration int `json:"request duration"` + RequestMethod string `json:"request method"` + RequestURI string `json:"request uri"` + ContentLength int `json:"content length"` + User string `json:"user"` + Script string `json:"script"` + LastRequestCPU float64 `json:"last request cpu"` + LastRequestMemory float64 `json:"last request memory"` + } `json:"processes"` +} + type metric map[string]int64 type poolStat map[string]metric type phpfpm struct { - Urls []string - Timeout config.Duration - tls.ClientConfig + Format string `toml:"format"` + Timeout config.Duration `toml:"timeout"` + Urls []string `toml:"urls"` + tls.ClientConfig client *http.Client + log telegraf.Logger } func (*phpfpm) SampleConfig() string { @@ -61,6 +96,15 @@ func (p *phpfpm) Init() error { return err } + switch p.Format { + case "": + p.Format = "status" + case "status", "json": + // both valid + default: + return fmt.Errorf("invalid mode: %s", p.Format) + } + p.client = &http.Client{ Transport: &http.Transport{ TLSClientConfig: tlsCfg, @@ -158,7 +202,7 @@ func (p *phpfpm) gatherFcgi(fcgi *conn, statusPath string, acc telegraf.Accumula }, "/"+statusPath) if len(fpmErr) == 0 && err == nil { - importMetric(bytes.NewReader(fpmOutput), acc, addr) + p.importMetric(bytes.NewReader(fpmOutput), acc, addr) return nil } return fmt.Errorf("unable parse phpfpm status, error: %s; %w", string(fpmErr), err) @@ -186,12 +230,20 @@ func (p *phpfpm) gatherHTTP(addr string, acc telegraf.Accumulator) error { return fmt.Errorf("unable to get valid stat result from %q: %w", addr, err) } - importMetric(res.Body, acc, addr) + p.importMetric(res.Body, acc, addr) return nil } // Import stat data into Telegraf system -func importMetric(r io.Reader, acc telegraf.Accumulator, addr string) { +func (p *phpfpm) importMetric(r io.Reader, acc telegraf.Accumulator, addr string) { + if p.Format == "json" { + p.parseJSON(r, acc, addr) + } else { + parseLines(r, acc, addr) + } +} + +func parseLines(r io.Reader, acc telegraf.Accumulator, addr string) { stats := make(poolStat) var currentPool string @@ -245,6 +297,55 @@ func importMetric(r io.Reader, acc telegraf.Accumulator, addr string) { } } +func (p *phpfpm) parseJSON(r io.Reader, acc telegraf.Accumulator, addr string) { + var metrics JSONMetrics + if err := json.NewDecoder(r).Decode(&metrics); err != nil { + p.log.Errorf("Unable to decode JSON response: %s", err) + return + } + timestamp := time.Now() + + tags := map[string]string{ + "pool": metrics.Pool, + "url": addr, + } + fields := map[string]any{ + "start_since": metrics.StartSince, + "accepted_conn": metrics.AcceptedConn, + "listen_queue": metrics.ListenQueue, + "max_listen_queue": metrics.MaxListenQueue, + "listen_queue_len": metrics.ListenQueueLen, + "idle_processes": metrics.IdleProcesses, + "active_processes": metrics.ActiveProcesses, + "total_processes": metrics.TotalProcesses, + "max_active_processes": metrics.MaxActiveProcesses, + "max_children_reached": metrics.MaxChildrenReached, + "slow_requests": metrics.SlowRequests, + } + acc.AddFields("phpfpm", fields, tags, timestamp) + + for _, process := range metrics.Processes { + tags := map[string]string{ + "pool": metrics.Pool, + "url": addr, + "user": process.User, + "request_uri": process.RequestURI, + "request_method": process.RequestMethod, + "script": process.Script, + } + fields := map[string]any{ + "state": process.State, + "start_time": process.StartTime, + "requests": process.Requests, + "request_duration": process.RequestDuration, + "content_length": process.ContentLength, + "last_request_cpu": process.LastRequestCPU, + "last_request_memory": process.LastRequestMemory, + } + acc.AddFields("phpfpm_process", fields, tags, timestamp) + } +} + func expandUrls(urls []string) ([]string, error) { addrs := make([]string, 0, len(urls)) for _, address := range urls { diff --git a/plugins/inputs/phpfpm/phpfpm_test.go b/plugins/inputs/phpfpm/phpfpm_test.go index 073cf9b1e0f5b..c53a1a6fe705b 100644 --- a/plugins/inputs/phpfpm/phpfpm_test.go +++ b/plugins/inputs/phpfpm/phpfpm_test.go @@ -13,11 +13,13 @@ import ( "net/http" "net/http/fcgi" "net/http/httptest" + "os" "strconv" "testing" "github.com/stretchr/testify/require" + "github.com/influxdata/telegraf/plugins/parsers/influx" "github.com/influxdata/telegraf/testutil" ) @@ -73,6 +75,34 @@ func TestPhpFpmGeneratesMetrics_From_Http(t *testing.T) { acc.AssertContainsTaggedFields(t, "phpfpm", fields, tags) } +func TestPhpFpmGeneratesJSONMetrics_From_Http(t *testing.T) { + outputSampleJSON, err := os.ReadFile("testdata/phpfpm.json") + require.NoError(t, err) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/json") + w.Header().Set("Content-Length", strconv.Itoa(len(outputSampleJSON))) + _, err := fmt.Fprint(w, string(outputSampleJSON)) + require.NoError(t, err) + })) + defer server.Close() + + parser := &influx.Parser{} + require.NoError(t, parser.Init()) + expected, err := testutil.ParseMetricsFromFile("testdata/expected.out", parser) + require.NoError(t, err) + + input := &phpfpm{ + Urls: []string{server.URL + "?full&json"}, + Format: "json", + log: testutil.Logger{}, + } + require.NoError(t, input.Init()) + + var acc testutil.Accumulator + require.NoError(t, acc.GatherError(input.Gather)) + testutil.RequireMetricsEqual(t, expected, acc.GetTelegrafMetrics(), testutil.IgnoreTime(), testutil.IgnoreTags("url")) +} + func TestPhpFpmGeneratesMetrics_From_Fcgi(t *testing.T) { // Let OS find an available port tcp, err := net.Listen("tcp", "127.0.0.1:0") diff --git a/plugins/inputs/phpfpm/sample.conf b/plugins/inputs/phpfpm/sample.conf index 62dbf501e53f2..86e48e375788f 100644 --- a/plugins/inputs/phpfpm/sample.conf +++ b/plugins/inputs/phpfpm/sample.conf @@ -23,6 +23,12 @@ ## urls = ["http://192.168.1.20/status", "/tmp/fpm.sock"] urls = ["http://localhost/status"] + ## Format of stats to parse, set to "status" or "json" + ## If the user configures the URL to return JSON (e.g. + ## http://localhost/status?json), set to JSON. Otherwise, will attempt to + ## parse line-by-line. The JSON mode will produce additonal metrics. + # format = "status" + ## Duration allowed to complete HTTP requests. # timeout = "5s" diff --git a/plugins/inputs/phpfpm/testdata/expected.out b/plugins/inputs/phpfpm/testdata/expected.out new file mode 100644 index 0000000000000..373e15c3fc98c --- /dev/null +++ b/plugins/inputs/phpfpm/testdata/expected.out @@ -0,0 +1,11 @@ +phpfpm,pool=www,url=http://127.0.0.1:44637?full&json accepted_conn=3879i,active_processes=1i,idle_processes=9i,listen_queue=0i,listen_queue_len=0i,max_active_processes=3i,max_children_reached=0i,max_listen_queue=0i,slow_requests=0i,start_since=4901i,total_processes=10i +phpfpm_process,pool=www,request_method=GET,request_uri=/fpm-status?json&full,script=-,url=http://127.0.0.1:44637?full&json,user=- content_length=0i,last_request_cpu=0,last_request_memory=0,request_duration=159i,requests=386i,start_time=1702044927i,state="Running" +phpfpm_process,pool=www,request_method=GET,request_uri=/fpm-status,script=-,url=http://127.0.0.1:44637?full&json,user=- content_length=0i,last_request_cpu=0,last_request_memory=2097152,request_duration=174i,requests=390i,start_time=1702044927i,state="Idle" +phpfpm_process,pool=www,request_method=GET,request_uri=/index.php,script=script.php,url=http://127.0.0.1:44637?full&json,user=- content_length=0i,last_request_cpu=104.93,last_request_memory=2097152,request_duration=9530i,requests=389i,start_time=1702044927i,state="Idle" +phpfpm_process,pool=www,request_method=GET,request_uri=/ping,script=-,url=http://127.0.0.1:44637?full&json,user=- content_length=0i,last_request_cpu=0,last_request_memory=2097152,request_duration=127i,requests=399i,start_time=1702044927i,state="Idle" +phpfpm_process,pool=www,request_method=GET,request_uri=/index.php,script=script.php,url=http://127.0.0.1:44637?full&json,user=- content_length=0i,last_request_cpu=0,last_request_memory=2097152,request_duration=9713i,requests=382i,start_time=1702044927i,state="Idle" +phpfpm_process,pool=www,request_method=GET,request_uri=/ping,script=-,url=http://127.0.0.1:44637?full&json,user=- content_length=0i,last_request_cpu=0,last_request_memory=2097152,request_duration=133i,requests=383i,start_time=1702044927i,state="Idle" +phpfpm_process,pool=www,request_method=GET,request_uri=/fpm-status?json,script=-,url=http://127.0.0.1:44637?full&json,user=- content_length=0i,last_request_cpu=0,last_request_memory=2097152,request_duration=154i,requests=381i,start_time=1702044927i,state="Idle" +phpfpm_process,pool=www,request_method=GET,request_uri=/ping,script=-,url=http://127.0.0.1:44637?full&json,user=- content_length=0i,last_request_cpu=0,last_request_memory=2097152,request_duration=108i,requests=397i,start_time=1702044927i,state="Idle" +phpfpm_process,pool=www,request_method=GET,request_uri=/index.php,script=script.php,url=http://127.0.0.1:44637?full&json,user=- content_length=0i,last_request_cpu=110.28,last_request_memory=2097152,request_duration=9068i,requests=381i,start_time=1702044927i,state="Idle" +phpfpm_process,pool=www,request_method=GET,request_uri=/index.php,script=script.php,url=http://127.0.0.1:44637?full&json,user=- content_length=0i,last_request_cpu=64.27,last_request_memory=2097152,request_duration=15559i,requests=391i,start_time=1702044927i,state="Idle" diff --git a/plugins/inputs/phpfpm/testdata/phpfpm.json b/plugins/inputs/phpfpm/testdata/phpfpm.json new file mode 100644 index 0000000000000..2826e6a8b1c4a --- /dev/null +++ b/plugins/inputs/phpfpm/testdata/phpfpm.json @@ -0,0 +1,168 @@ +{ + "pool": "www", + "process manager": "static", + "start time": 1702044927, + "start since": 4901, + "accepted conn": 3879, + "listen queue": 0, + "max listen queue": 0, + "listen queue len": 0, + "idle processes": 9, + "active processes": 1, + "total processes": 10, + "max active processes": 3, + "max children reached": 0, + "slow requests": 0, + "processes": [ + { + "pid": 583, + "state": "Running", + "start time": 1702044927, + "start since": 4901, + "requests": 386, + "request duration": 159, + "request method": "GET", + "request uri": "/fpm-status?json&full", + "content length": 0, + "user": "-", + "script": "-", + "last request cpu": 0, + "last request memory": 0 + }, + { + "pid": 584, + "state": "Idle", + "start time": 1702044927, + "start since": 4901, + "requests": 390, + "request duration": 174, + "request method": "GET", + "request uri": "/fpm-status", + "content length": 0, + "user": "-", + "script": "-", + "last request cpu": 0, + "last request memory": 2097152 + }, + { + "pid": 585, + "state": "Idle", + "start time": 1702044927, + "start since": 4901, + "requests": 389, + "request duration": 9530, + "request method": "GET", + "request uri": "/index.php", + "content length": 0, + "user": "-", + "script": "script.php", + "last request cpu": 104.93, + "last request memory": 2097152 + }, + { + "pid": 586, + "state": "Idle", + "start time": 1702044927, + "start since": 4901, + "requests": 399, + "request duration": 127, + "request method": "GET", + "request uri": "/ping", + "content length": 0, + "user": "-", + "script": "-", + "last request cpu": 0, + "last request memory": 2097152 + }, + { + "pid": 587, + "state": "Idle", + "start time": 1702044927, + "start since": 4901, + "requests": 382, + "request duration": 9713, + "request method": "GET", + "request uri": "/index.php", + "content length": 0, + "user": "-", + "script": "script.php", + "last request cpu": 0, + "last request memory": 2097152 + }, + { + "pid": 588, + "state": "Idle", + "start time": 1702044927, + "start since": 4901, + "requests": 383, + "request duration": 133, + "request method": "GET", + "request uri": "/ping", + "content length": 0, + "user": "-", + "script": "-", + "last request cpu": 0, + "last request memory": 2097152 + }, + { + "pid": 589, + "state": "Idle", + "start time": 1702044927, + "start since": 4901, + "requests": 381, + "request duration": 154, + "request method": "GET", + "request uri": "/fpm-status?json", + "content length": 0, + "user": "-", + "script": "-", + "last request cpu": 0, + "last request memory": 2097152 + }, + { + "pid": 590, + "state": "Idle", + "start time": 1702044927, + "start since": 4901, + "requests": 397, + "request duration": 108, + "request method": "GET", + "request uri": "/ping", + "content length": 0, + "user": "-", + "script": "-", + "last request cpu": 0, + "last request memory": 2097152 + }, + { + "pid": 591, + "state": "Idle", + "start time": 1702044927, + "start since": 4901, + "requests": 381, + "request duration": 9068, + "request method": "GET", + "request uri": "/index.php", + "content length": 0, + "user": "-", + "script": "script.php", + "last request cpu": 110.28, + "last request memory": 2097152 + }, + { + "pid": 592, + "state": "Idle", + "start time": 1702044927, + "start since": 4901, + "requests": 391, + "request duration": 15559, + "request method": "GET", + "request uri": "/index.php", + "content length": 0, + "user": "-", + "script": "script.php", + "last request cpu": 64.27, + "last request memory": 2097152 + } + ] +} From 24330b4946d3bf72eb6fc87c18dab6cde691d61c Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Wed, 13 Dec 2023 07:18:31 -0700 Subject: [PATCH 2/2] Update plugins/inputs/phpfpm/phpfpm.go Co-authored-by: Sven Rebhan <36194019+srebhan@users.noreply.github.com> --- plugins/inputs/phpfpm/phpfpm.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/inputs/phpfpm/phpfpm.go b/plugins/inputs/phpfpm/phpfpm.go index 9521b936165cb..197c768600a14 100644 --- a/plugins/inputs/phpfpm/phpfpm.go +++ b/plugins/inputs/phpfpm/phpfpm.go @@ -102,7 +102,7 @@ func (p *phpfpm) Init() error { case "status", "json": // both valid default: - return fmt.Errorf("invalid mode: %s", p.Format) + return fmt.Errorf("invalid format: %s", p.Format) } p.client = &http.Client{