Skip to content

Commit

Permalink
Parse JSON-style reports in golang side
Browse files Browse the repository at this point in the history
Cherry-pick of existing commit.
orig-pr: #18159
orig-commit: 86308c3
orig-commit-author: twalluxio <139105284+twalluxio@users.noreply.github.com>

			pr-link: #18166
			change-id: cid-deb6e74552de9afcf45391c6c230a9fe00785e37
  • Loading branch information
twalluxio authored Sep 22, 2023
1 parent be6974c commit ba7fedf
Show file tree
Hide file tree
Showing 9 changed files with 177 additions and 114 deletions.
108 changes: 101 additions & 7 deletions cli/src/alluxio.org/cli/cmd/info/report.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,17 @@
package info

import (
"bytes"
"encoding/json"
"fmt"
"io"
"math"
"os"
"strconv"
"strings"
"time"

"github.com/iancoleman/orderedmap"
"github.com/palantir/stacktrace"
"github.com/spf13/cobra"

Expand All @@ -32,7 +40,7 @@ var Report = &ReportCommand{

type ReportCommand struct {
*env.BaseJavaCommand
format string
raw bool
}

func (c *ReportCommand) Base() *env.BaseJavaCommand {
Expand All @@ -57,8 +65,8 @@ Defaults to summary if no arg is provided
return c.Run(args)
},
})
cmd.Flags().StringVar(&c.format, "format", "json",
"Set output format, any of [json, yaml]")
cmd.Flags().BoolVar(&c.raw, "raw", false,
"Output raw JSON data instead of human-readable format for bytes, datetime, and duration.")
return cmd
}

Expand All @@ -80,9 +88,95 @@ func (c *ReportCommand) Run(args []string) error {
}
reportArg = args[0]
}
// TODO: output all in a serializable format and filter/trim as specified by flags
if c.format != "json" && c.format != "yaml" {
return stacktrace.NewError("Invalid format %v, must be one of [json, yaml]", c.format)

buf := &bytes.Buffer{}
if err := c.RunWithIO([]string{reportArg}, nil, buf, os.Stderr); err != nil {
io.Copy(os.Stdout, buf)
return err
}

obj := orderedmap.New()
if err := obj.UnmarshalJSON(buf.Bytes()); err != nil {
return stacktrace.Propagate(err, "error unmarshalling json from java command")
}

if !c.raw {
newObj := orderedmap.New()
for _, key := range obj.Keys() {
if val, ok := obj.Get(key); ok {
k, v := processKeyValuePair(key, val)
newObj.Set(k, v)
}
}
obj = newObj
}

prettyJson, err := json.MarshalIndent(obj, "", " ")
if err != nil {
return stacktrace.Propagate(err, "error marshalling json to pretty format")
}
os.Stdout.Write(append(prettyJson, '\n'))
return nil
}

func convertMsToDatetime(timeMs float64) string {
tm := time.Unix(int64(timeMs)/1000, 0)
return tm.Format(time.RFC3339)
}

func convertMsToDuration(timeMs float64) string {
duration := time.Duration(int64(timeMs)) * time.Millisecond
days := duration / (24 * time.Hour)
duration = duration % (24 * time.Hour)
hours := duration / time.Hour
duration = duration % time.Hour
minutes := duration / time.Minute
seconds := duration % time.Minute / time.Second
return fmt.Sprintf("%dd %02dh%02dm%02ds", days, hours, minutes, seconds)
}

func convertBytesToString(bytes float64) string {
const unit = 1024
suffixes := []string{"B", "KB", "MB", "GB", "TB", "PB"}
exp, n := 0, int64(bytes)
for n > 5*unit && exp < len(suffixes)-1 {
n /= unit
exp++
}
value := bytes / math.Pow(unit, float64(exp))
size := strconv.FormatFloat(value, 'f', 2, 64)
return size + suffixes[exp]
}

func processKeyValuePair(key string, data interface{}) (string, interface{}) {
switch value := data.(type) {
case float64:
if strings.HasSuffix(key, "Time") {
return strings.TrimSuffix(key, "Time"), convertMsToDatetime(value)
} else if strings.HasSuffix(key, "Duration") {
return strings.TrimSuffix(key, "Duration"), convertMsToDuration(value)
} else if strings.HasSuffix(key, "Bytes") {
return strings.TrimSuffix(key, "Bytes"), convertBytesToString(value)
} else {
return key, value
}
case []interface{}:
array := make([]interface{}, len(value))
for i, item := range value {
_, processedItem := processKeyValuePair(key, item)
array[i] = processedItem
}
return key, array
case orderedmap.OrderedMap:
processedMap := orderedmap.New()
for _, key := range value.Keys() {
if val, ok := value.Get(key); ok {
k, v := processKeyValuePair(key, val)
processedMap.Set(k, v)
}
}
return key, processedMap
default:
return key, value
}
return c.Base().RunAndFormat(c.format, nil, []string{reportArg})
}
1 change: 1 addition & 0 deletions cli/src/alluxio.org/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ module alluxio.org
go 1.18

require (
github.com/iancoleman/orderedmap v0.3.0
github.com/palantir/stacktrace v0.0.0-20161112013806-78658fd2d177
github.com/shirou/gopsutil v3.21.11+incompatible
github.com/sirupsen/logrus v1.9.0
Expand Down
2 changes: 2 additions & 0 deletions cli/src/alluxio.org/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,8 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/iancoleman/orderedmap v0.3.0 h1:5cbR2grmZR/DiVt+VJopEhtVs9YGInGIxAoMJn+Ichc=
github.com/iancoleman/orderedmap v0.3.0/go.mod h1:XuLcCUkdL5owUCQeF2Ue9uuw1EptkJDkXXS7VoV7XGE=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,7 @@

import java.io.IOException;
import java.io.PrintStream;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.time.format.FormatStyle;
import java.util.List;
import java.util.Locale;

/**
* Prints job service metric information.
Expand All @@ -53,10 +49,6 @@ public JobServiceMetricsCommand(JobMasterClient JobMasterClient, PrintStream pri
mDateFormatPattern = dateFormatPattern;
}

public static final DateTimeFormatter DATETIME_FORMAT =
DateTimeFormatter.ofLocalizedDateTime(FormatStyle.SHORT).ofPattern("yyyyMMdd-HHmmss")
.withLocale(Locale.getDefault()).withZone(ZoneId.systemDefault());

/**
* Runs a job services report metrics command.
*
Expand All @@ -68,8 +60,8 @@ public int run() throws IOException {
List<JobWorkerHealth> allWorkerHealth = mJobMasterClient.getAllWorkerHealth();
JobServiceSummary jobServiceSummary = mJobMasterClient.getJobServiceSummary();

JobServiceOutput jobServiceInfo = new JobServiceOutput(allMasterStatus, allWorkerHealth,
jobServiceSummary, mDateFormatPattern);
JobServiceOutput jobServiceInfo = new JobServiceOutput(
allMasterStatus, allWorkerHealth, jobServiceSummary);
try {
String json = objectMapper.writeValueAsString(jobServiceInfo);
mPrintStream.println(json);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
import alluxio.job.wire.JobServiceSummary;
import alluxio.job.wire.JobWorkerHealth;
import alluxio.job.wire.StatusSummary;
import alluxio.util.CommonUtils;

import java.util.ArrayList;
import java.util.List;
Expand All @@ -36,15 +35,15 @@ private static class SerializableJobMasterStatus {
private String mHost;
private int mPort;
private String mState;
private String mStartTime;
private long mStartTime;
private String mVersion;
private String mRevision;

public SerializableJobMasterStatus(JobMasterStatus jobMasterStatus, String dateFormat) {
public SerializableJobMasterStatus(JobMasterStatus jobMasterStatus) {
mHost = jobMasterStatus.getMasterAddress().getHost();
mPort = jobMasterStatus.getMasterAddress().getRpcPort();
mState = jobMasterStatus.getState();
mStartTime = CommonUtils.convertMsToDate(jobMasterStatus.getStartTime(), dateFormat);
mStartTime = jobMasterStatus.getStartTime();
mVersion = jobMasterStatus.getVersion().getVersion();
mRevision = jobMasterStatus.getVersion().getRevision();
}
Expand Down Expand Up @@ -73,11 +72,11 @@ public void setState(String state) {
mState = state;
}

public String getStartTime() {
public long getStartTime() {
return mStartTime;
}

public void setStartTime(String startTime) {
public void setStartTime(long startTime) {
mStartTime = startTime;
}

Expand Down Expand Up @@ -201,24 +200,24 @@ public void setCount(long count) {
}

private static class SerializableJobInfo {
private String mTimestamp;
private long mLastUpdatedTime;
private long mId;
private String mName;
private String mStatus;

public SerializableJobInfo(JobInfo jobInfo, String dateFormat) {
mTimestamp = CommonUtils.convertMsToDate(jobInfo.getLastUpdated(), dateFormat);
public SerializableJobInfo(JobInfo jobInfo) {
mLastUpdatedTime = jobInfo.getLastUpdated();
mId = jobInfo.getId();
mName = jobInfo.getName();
mStatus = jobInfo.getStatus().toString();
}

public String getTimestamp() {
return mTimestamp;
public long getLastUpdatedTime() {
return mLastUpdatedTime;
}

public void setTimestamp(String timestamp) {
mTimestamp = timestamp;
public void setLastUpdatedTime(long lastUpdatedTime) {
mLastUpdatedTime = lastUpdatedTime;
}

public long getId() {
Expand Down Expand Up @@ -252,15 +251,13 @@ public void setStatus(String status) {
* @param allMasterStatus status of all masters
* @param allWorkerHealth health info of all workers
* @param jobServiceSummary summary of job service
* @param dateFormat specify the pattern of dates
*/
public JobServiceOutput(List<JobMasterStatus> allMasterStatus,
List<JobWorkerHealth> allWorkerHealth,
JobServiceSummary jobServiceSummary,
String dateFormat) {
JobServiceSummary jobServiceSummary) {
mMasterStatus = new ArrayList<>();
for (JobMasterStatus masterStatus : allMasterStatus) {
mMasterStatus.add(new SerializableJobMasterStatus(masterStatus, dateFormat));
mMasterStatus.add(new SerializableJobMasterStatus(masterStatus));
}

mWorkerHealth = new ArrayList<>();
Expand All @@ -276,13 +273,13 @@ public JobServiceOutput(List<JobMasterStatus> allMasterStatus,
mRecentFailedJobs = new ArrayList<>();
mLongestRunningJobs = new ArrayList<>();
for (JobInfo jobInfo : jobServiceSummary.getRecentActivities()) {
mRecentModifiedJobs.add(new SerializableJobInfo(jobInfo, dateFormat));
mRecentModifiedJobs.add(new SerializableJobInfo(jobInfo));
}
for (JobInfo jobInfo : jobServiceSummary.getRecentFailures()) {
mRecentFailedJobs.add(new SerializableJobInfo(jobInfo, dateFormat));
mRecentFailedJobs.add(new SerializableJobInfo(jobInfo));
}
for (JobInfo jobInfo : jobServiceSummary.getLongestRunning()) {
mLongestRunningJobs.add(new SerializableJobInfo(jobInfo, dateFormat));
mLongestRunningJobs.add(new SerializableJobInfo(jobInfo));
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ public int run() throws IOException {
BlockMasterInfo blockMasterInfo = mBlockMasterClient.getBlockMasterInfo(blockMasterInfoFilter);

ObjectMapper objectMapper = new ObjectMapper();
SummaryOutput summaryInfo = new SummaryOutput(masterInfo, blockMasterInfo, mDateFormatPattern);
SummaryOutput summaryInfo = new SummaryOutput(masterInfo, blockMasterInfo);
try {
String json = objectMapper.writeValueAsString(summaryInfo);
mPrintStream.println(json);
Expand Down
Loading

0 comments on commit ba7fedf

Please sign in to comment.