Skip to content

Commit

Permalink
fix(inputs.php-fpm): Parse JSON output
Browse files Browse the repository at this point in the history 
fixes: #14421
  • Loading branch information
@powersj
powersj committed Dec 12, 2023
1 parent 2b731c2 commit 35e9b5d
Show file tree
Hide file tree
Showing 6 changed files with 361 additions and 6 deletions.
39 changes: 39 additions & 0 deletions plugins/inputs/phpfpm/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,12 @@ See the [CONFIGURATION.md][CONFIGURATION.md] for more details.
## urls = ["http://192.168.1.20/status", "/tmp/fpm.sock"]
urls = ["http://localhost/status"]

## Format of stats to parse, set to "status" or "json"
## If the user configures the URL to return JSON (e.g.
## http://localhost/status?json), set to JSON. Otherwise, will attempt to
## parse line-by-line. The JSON mode will produce additonal metrics.
# format = "status"

## Duration allowed to complete HTTP requests.
# timeout = "5s"

Expand Down Expand Up @@ -70,6 +76,23 @@ host, and socket path is accessible to telegraf user.
- max_active_processes
- max_children_reached
- slow_requests
- phpfpm_process
- tags:
- pool
- request method
- request uri
- script
- url
- user
- fields:
- content length
- last request cpu
- last request memory
- request duration
- requests
- start time
- start since
- state

## Example Output

Expand All @@ -78,3 +101,19 @@ phpfpm,pool=www accepted_conn=13i,active_processes=2i,idle_processes=1i,listen_q
phpfpm,pool=www2 accepted_conn=12i,active_processes=1i,idle_processes=2i,listen_queue=0i,listen_queue_len=0i,max_active_processes=2i,max_children_reached=0i,max_listen_queue=0i,slow_requests=0i,total_processes=3i 1453011293083691422
phpfpm,pool=www3 accepted_conn=11i,active_processes=1i,idle_processes=2i,listen_queue=0i,listen_queue_len=0i,max_active_processes=2i,max_children_reached=0i,max_listen_queue=0i,slow_requests=0i,total_processes=3i 1453011293083691658
```

With the JSON output, additional metrics around processes are generated:

```text
phpfpm,pool=www,url=http://127.0.0.1:44637?full&json accepted_conn=3879i,active_processes=1i,idle_processes=9i,listen_queue=0i,listen_queue_len=0i,max_active_processes=3i,max_children_reached=0i,max_listen_queue=0i,slow_requests=0i,start_since=4901i,total_processes=10i
phpfpm_process,pool=www,request_method=GET,request_uri=/fpm-status?json&full,script=-,url=http://127.0.0.1:44637?full&json,user=- content_length=0i,last_request_cpu=0,last_request_memory=0,request_duration=159i,requests=386i,start_time=1702044927i,state="Running"
phpfpm_process,pool=www,request_method=GET,request_uri=/fpm-status,script=-,url=http://127.0.0.1:44637?full&json,user=- content_length=0i,last_request_cpu=0,last_request_memory=2097152,request_duration=174i,requests=390i,start_time=1702044927i,state="Idle"
phpfpm_process,pool=www,request_method=GET,request_uri=/index.php,script=script.php,url=http://127.0.0.1:44637?full&json,user=- content_length=0i,last_request_cpu=104.93,last_request_memory=2097152,request_duration=9530i,requests=389i,start_time=1702044927i,state="Idle"
phpfpm_process,pool=www,request_method=GET,request_uri=/ping,script=-,url=http://127.0.0.1:44637?full&json,user=- content_length=0i,last_request_cpu=0,last_request_memory=2097152,request_duration=127i,requests=399i,start_time=1702044927i,state="Idle"
phpfpm_process,pool=www,request_method=GET,request_uri=/index.php,script=script.php,url=http://127.0.0.1:44637?full&json,user=- content_length=0i,last_request_cpu=0,last_request_memory=2097152,request_duration=9713i,requests=382i,start_time=1702044927i,state="Idle"
phpfpm_process,pool=www,request_method=GET,request_uri=/ping,script=-,url=http://127.0.0.1:44637?full&json,user=- content_length=0i,last_request_cpu=0,last_request_memory=2097152,request_duration=133i,requests=383i,start_time=1702044927i,state="Idle"
phpfpm_process,pool=www,request_method=GET,request_uri=/fpm-status?json,script=-,url=http://127.0.0.1:44637?full&json,user=- content_length=0i,last_request_cpu=0,last_request_memory=2097152,request_duration=154i,requests=381i,start_time=1702044927i,state="Idle"
phpfpm_process,pool=www,request_method=GET,request_uri=/ping,script=-,url=http://127.0.0.1:44637?full&json,user=- content_length=0i,last_request_cpu=0,last_request_memory=2097152,request_duration=108i,requests=397i,start_time=1702044927i,state="Idle"
phpfpm_process,pool=www,request_method=GET,request_uri=/index.php,script=script.php,url=http://127.0.0.1:44637?full&json,user=- content_length=0i,last_request_cpu=110.28,last_request_memory=2097152,request_duration=9068i,requests=381i,start_time=1702044927i,state="Idle"
phpfpm_process,pool=www,request_method=GET,request_uri=/index.php,script=script.php,url=http://127.0.0.1:44637?full&json,user=- content_length=0i,last_request_cpu=64.27,last_request_memory=2097152,request_duration=15559i,requests=391i,start_time=1702044927i,state="Idle"
```
113 changes: 107 additions & 6 deletions plugins/inputs/phpfpm/phpfpm.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"bufio"
"bytes"
_ "embed"
"encoding/json"
"fmt"
"io"
"net/http"
Expand Down Expand Up @@ -40,15 +41,49 @@ const (
PfSlowRequests = "slow requests"
)

type JSONMetrics struct {
Pool string `json:"pool"`
ProcessManager string `json:"process manager"`
StartTime int `json:"start time"`
StartSince int `json:"start since"`
AcceptedConn int `json:"accepted conn"`
ListenQueue int `json:"listen queue"`
MaxListenQueue int `json:"max listen queue"`
ListenQueueLen int `json:"listen queue len"`
IdleProcesses int `json:"idle processes"`
ActiveProcesses int `json:"active processes"`
TotalProcesses int `json:"total processes"`
MaxActiveProcesses int `json:"max active processes"`
MaxChildrenReached int `json:"max children reached"`
SlowRequests int `json:"slow requests"`
Processes []struct {
Pid int `json:"pid"`
State string `json:"state"`
StartTime int `json:"start time"`
StartSince int `json:"start since"`
Requests int `json:"requests"`
RequestDuration int `json:"request duration"`
RequestMethod string `json:"request method"`
RequestURI string `json:"request uri"`
ContentLength int `json:"content length"`
User string `json:"user"`
Script string `json:"script"`
LastRequestCPU float64 `json:"last request cpu"`
LastRequestMemory float64 `json:"last request memory"`
} `json:"processes"`
}

type metric map[string]int64
type poolStat map[string]metric

type phpfpm struct {
Urls []string
Timeout config.Duration
tls.ClientConfig
Format string `toml:"format"`
Timeout config.Duration `toml:"timeout"`
Urls []string `toml:"urls"`

tls.ClientConfig
client *http.Client
log telegraf.Logger
}

func (*phpfpm) SampleConfig() string {
Expand All @@ -61,6 +96,15 @@ func (p *phpfpm) Init() error {
return err
}

switch p.Format {
case "":
p.Format = "status"
case "status", "json":
// both valid
default:
return fmt.Errorf("invalid mode: %s", p.Format)
}

p.client = &http.Client{
Transport: &http.Transport{
TLSClientConfig: tlsCfg,
Expand Down Expand Up @@ -158,7 +202,7 @@ func (p *phpfpm) gatherFcgi(fcgi *conn, statusPath string, acc telegraf.Accumula
}, "/"+statusPath)

if len(fpmErr) == 0 && err == nil {
importMetric(bytes.NewReader(fpmOutput), acc, addr)
p.importMetric(bytes.NewReader(fpmOutput), acc, addr)
return nil
}
return fmt.Errorf("unable parse phpfpm status, error: %s; %w", string(fpmErr), err)
Expand Down Expand Up @@ -186,12 +230,20 @@ func (p *phpfpm) gatherHTTP(addr string, acc telegraf.Accumulator) error {
return fmt.Errorf("unable to get valid stat result from %q: %w", addr, err)
}

importMetric(res.Body, acc, addr)
p.importMetric(res.Body, acc, addr)
return nil
}

// Import stat data into Telegraf system
func importMetric(r io.Reader, acc telegraf.Accumulator, addr string) {
func (p *phpfpm) importMetric(r io.Reader, acc telegraf.Accumulator, addr string) {
if p.Format == "json" {
p.parseJSON(r, acc, addr)
} else {
parseLines(r, acc, addr)
}
}

func parseLines(r io.Reader, acc telegraf.Accumulator, addr string) {
stats := make(poolStat)
var currentPool string

Expand Down Expand Up @@ -245,6 +297,55 @@ func importMetric(r io.Reader, acc telegraf.Accumulator, addr string) {
}
}

func (p *phpfpm) parseJSON(r io.Reader, acc telegraf.Accumulator, addr string) {
var metrics JSONMetrics
if err := json.NewDecoder(r).Decode(&metrics); err != nil {
p.log.Errorf("Unable to decode JSON response: %s", err)
return
}
timestamp := time.Now()

tags := map[string]string{
"pool": metrics.Pool,
"url": addr,
}
fields := map[string]any{
"start_since": metrics.StartSince,
"accepted_conn": metrics.AcceptedConn,
"listen_queue": metrics.ListenQueue,
"max_listen_queue": metrics.MaxListenQueue,
"listen_queue_len": metrics.ListenQueueLen,
"idle_processes": metrics.IdleProcesses,
"active_processes": metrics.ActiveProcesses,
"total_processes": metrics.TotalProcesses,
"max_active_processes": metrics.MaxActiveProcesses,
"max_children_reached": metrics.MaxChildrenReached,
"slow_requests": metrics.SlowRequests,
}
acc.AddFields("phpfpm", fields, tags, timestamp)

for _, process := range metrics.Processes {
tags := map[string]string{
"pool": metrics.Pool,
"url": addr,
"user": process.User,
"request_uri": process.RequestURI,
"request_method": process.RequestMethod,
"script": process.Script,
}
fields := map[string]any{
"state": process.State,
"start_time": process.StartTime,
"requests": process.Requests,
"request_duration": process.RequestDuration,
"content_length": process.ContentLength,
"last_request_cpu": process.LastRequestCPU,
"last_request_memory": process.LastRequestMemory,
}
acc.AddFields("phpfpm_process", fields, tags, timestamp)
}
}

func expandUrls(urls []string) ([]string, error) {
addrs := make([]string, 0, len(urls))
for _, address := range urls {
Expand Down
30 changes: 30 additions & 0 deletions plugins/inputs/phpfpm/phpfpm_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,13 @@ import (
"net/http"
"net/http/fcgi"
"net/http/httptest"
"os"
"strconv"
"testing"

"github.com/stretchr/testify/require"

"github.com/influxdata/telegraf/plugins/parsers/influx"
"github.com/influxdata/telegraf/testutil"
)

Expand Down Expand Up @@ -73,6 +75,34 @@ func TestPhpFpmGeneratesMetrics_From_Http(t *testing.T) {
acc.AssertContainsTaggedFields(t, "phpfpm", fields, tags)
}

func TestPhpFpmGeneratesJSONMetrics_From_Http(t *testing.T) {
outputSampleJSON, err := os.ReadFile("testdata/phpfpm.json")
require.NoError(t, err)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/json")
w.Header().Set("Content-Length", strconv.Itoa(len(outputSampleJSON)))
_, err := fmt.Fprint(w, string(outputSampleJSON))
require.NoError(t, err)
}))
defer server.Close()

parser := &influx.Parser{}
require.NoError(t, parser.Init())
expected, err := testutil.ParseMetricsFromFile("testdata/expected.out", parser)
require.NoError(t, err)

input := &phpfpm{
Urls: []string{server.URL + "?full&json"},
Format: "json",
log: testutil.Logger{},
}
require.NoError(t, input.Init())

var acc testutil.Accumulator
require.NoError(t, acc.GatherError(input.Gather))
testutil.RequireMetricsEqual(t, expected, acc.GetTelegrafMetrics(), testutil.IgnoreTime(), testutil.IgnoreTags("url"))
}

func TestPhpFpmGeneratesMetrics_From_Fcgi(t *testing.T) {
// Let OS find an available port
tcp, err := net.Listen("tcp", "127.0.0.1:0")
Expand Down
6 changes: 6 additions & 0 deletions plugins/inputs/phpfpm/sample.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,12 @@
## urls = ["http://192.168.1.20/status", "/tmp/fpm.sock"]
urls = ["http://localhost/status"]

## Format of stats to parse, set to "status" or "json"
## If the user configures the URL to return JSON (e.g.
## http://localhost/status?json), set to JSON. Otherwise, will attempt to
## parse line-by-line. The JSON mode will produce additonal metrics.
# format = "status"

## Duration allowed to complete HTTP requests.
# timeout = "5s"

Expand Down
11 changes: 11 additions & 0 deletions plugins/inputs/phpfpm/testdata/expected.out
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
phpfpm,pool=www,url=http://127.0.0.1:44637?full&json accepted_conn=3879i,active_processes=1i,idle_processes=9i,listen_queue=0i,listen_queue_len=0i,max_active_processes=3i,max_children_reached=0i,max_listen_queue=0i,slow_requests=0i,start_since=4901i,total_processes=10i
phpfpm_process,pool=www,request_method=GET,request_uri=/fpm-status?json&full,script=-,url=http://127.0.0.1:44637?full&json,user=- content_length=0i,last_request_cpu=0,last_request_memory=0,request_duration=159i,requests=386i,start_time=1702044927i,state="Running"
phpfpm_process,pool=www,request_method=GET,request_uri=/fpm-status,script=-,url=http://127.0.0.1:44637?full&json,user=- content_length=0i,last_request_cpu=0,last_request_memory=2097152,request_duration=174i,requests=390i,start_time=1702044927i,state="Idle"
phpfpm_process,pool=www,request_method=GET,request_uri=/index.php,script=script.php,url=http://127.0.0.1:44637?full&json,user=- content_length=0i,last_request_cpu=104.93,last_request_memory=2097152,request_duration=9530i,requests=389i,start_time=1702044927i,state="Idle"
phpfpm_process,pool=www,request_method=GET,request_uri=/ping,script=-,url=http://127.0.0.1:44637?full&json,user=- content_length=0i,last_request_cpu=0,last_request_memory=2097152,request_duration=127i,requests=399i,start_time=1702044927i,state="Idle"
phpfpm_process,pool=www,request_method=GET,request_uri=/index.php,script=script.php,url=http://127.0.0.1:44637?full&json,user=- content_length=0i,last_request_cpu=0,last_request_memory=2097152,request_duration=9713i,requests=382i,start_time=1702044927i,state="Idle"
phpfpm_process,pool=www,request_method=GET,request_uri=/ping,script=-,url=http://127.0.0.1:44637?full&json,user=- content_length=0i,last_request_cpu=0,last_request_memory=2097152,request_duration=133i,requests=383i,start_time=1702044927i,state="Idle"
phpfpm_process,pool=www,request_method=GET,request_uri=/fpm-status?json,script=-,url=http://127.0.0.1:44637?full&json,user=- content_length=0i,last_request_cpu=0,last_request_memory=2097152,request_duration=154i,requests=381i,start_time=1702044927i,state="Idle"
phpfpm_process,pool=www,request_method=GET,request_uri=/ping,script=-,url=http://127.0.0.1:44637?full&json,user=- content_length=0i,last_request_cpu=0,last_request_memory=2097152,request_duration=108i,requests=397i,start_time=1702044927i,state="Idle"
phpfpm_process,pool=www,request_method=GET,request_uri=/index.php,script=script.php,url=http://127.0.0.1:44637?full&json,user=- content_length=0i,last_request_cpu=110.28,last_request_memory=2097152,request_duration=9068i,requests=381i,start_time=1702044927i,state="Idle"
phpfpm_process,pool=www,request_method=GET,request_uri=/index.php,script=script.php,url=http://127.0.0.1:44637?full&json,user=- content_length=0i,last_request_cpu=64.27,last_request_memory=2097152,request_duration=15559i,requests=391i,start_time=1702044927i,state="Idle"
Loading

0 comments on commit 35e9b5d

Please sign in to comment.