Skip to content

Commit

Permalink
remove file pruning
Browse files Browse the repository at this point in the history
This can be done easily with the find command, no need to write it again.
  • Loading branch information
dnephin committed Nov 26, 2022
1 parent ff9bd18 commit d0adc24
Show file tree
Hide file tree
Showing 2 changed files with 11 additions and 98 deletions.
39 changes: 7 additions & 32 deletions cmd/tool/matrix/matrix.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,9 @@ func Run(name string, args []string) error {
}

type options struct {
pruneFilesMaxAgeDays uint
numPartitions uint
timingFilesPattern string
debug bool
numPartitions uint
timingFilesPattern string
debug bool

// shims for testing
stdin io.Reader
Expand All @@ -49,8 +48,6 @@ func setupFlags(name string) (*pflag.FlagSet, *options) {
flags.Usage = func() {
usage(os.Stdout, name, flags)
}
flags.UintVar(&opts.pruneFilesMaxAgeDays, "max-age-days", 0,
"timing files older than this value will be deleted")
flags.UintVar(&opts.numPartitions, "partitions", 0,
"number of parallel partitions to create in the test matrix")
flags.StringVar(&opts.timingFilesPattern, "timing-files", "",
Expand All @@ -67,11 +64,9 @@ func usage(out io.Writer, name string, flags *pflag.FlagSet) {
Read a list of packages from stdin and output a GitHub Actions matrix strategy
that splits the packages by previous run times to minimize overall CI runtime.
Example
echo -n "::set-output name=matrix::"
go list ./... | \
%[1]s --partitions 4 --timing-files ./*.log --max-age-days 10
%[1]s --timing-files ./*.log --partitions 4
The output of the command is a JSON object that can be used as the matrix
strategy for a test job.
Expand Down Expand Up @@ -100,7 +95,7 @@ func run(opts options) error {
return fmt.Errorf("failed to read packages from stdin: %v", err)
}

files, err := readAndPruneTimingReports(opts)
files, err := readTimingReports(opts)
if err != nil {
return fmt.Errorf("failed to read or delete timing files: %v", err)
}
Expand All @@ -124,7 +119,7 @@ func readPackages(stdin io.Reader) ([]string, error) {
return packages, scan.Err()
}

func readAndPruneTimingReports(opts options) ([]*os.File, error) {
func readTimingReports(opts options) ([]*os.File, error) {
fileNames, err := filepath.Glob(opts.timingFilesPattern)
if err != nil {
return nil, err
Expand All @@ -136,27 +131,7 @@ func readAndPruneTimingReports(opts options) ([]*os.File, error) {
if err != nil {
return nil, err
}

event, err := parseEvent(fh)
if err != nil {
return nil, fmt.Errorf("failed to read first event from %v: %v", fh.Name(), err)
}

age := time.Since(event.Time)
maxAge := time.Duration(opts.pruneFilesMaxAgeDays) * 24 * time.Hour
if opts.pruneFilesMaxAgeDays == 0 || age < maxAge {
if _, err := fh.Seek(0, io.SeekStart); err != nil {
return nil, fmt.Errorf("failed to reset file: %v", err)
}
files = append(files, fh)
continue
}

log.Infof("Removing %v because it is from %v", fh.Name(), event.Time.Format(time.RFC1123))
_ = fh.Close()
if err := os.Remove(fh.Name()); err != nil {
return nil, err
}
files = append(files, fh)
}

log.Infof("Found %v timing files in %v", len(files), opts.timingFilesPattern)
Expand Down
70 changes: 4 additions & 66 deletions cmd/tool/matrix/matrix_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ func TestBucketPackages(t *testing.T) {
}
}

func TestReadAndPruneTimingReports(t *testing.T) {
func TestReadTimingReports(t *testing.T) {
events := func(t *testing.T, start time.Time) string {
t.Helper()
var buf bytes.Buffer
Expand All @@ -140,12 +140,12 @@ func TestReadAndPruneTimingReports(t *testing.T) {
fs.WithFile("report3.log", events(t, now.Add(-49*time.Hour))),
fs.WithFile("report4.log", events(t, now.Add(-101*time.Hour))))

t.Run("no prune", func(t *testing.T) {
t.Run("match files", func(t *testing.T) {
opts := options{
timingFilesPattern: dir.Join("*.log"),
}

files, err := readAndPruneTimingReports(opts)
files, err := readTimingReports(opts)
assert.NilError(t, err)
defer closeFiles(files)
assert.Equal(t, len(files), 4)
Expand All @@ -167,33 +167,10 @@ func TestReadAndPruneTimingReports(t *testing.T) {
timingFilesPattern: dir.Join("*.json"),
}

files, err := readAndPruneTimingReports(opts)
files, err := readTimingReports(opts)
assert.NilError(t, err)
assert.Equal(t, len(files), 0)
})

t.Run("prune older than max age", func(t *testing.T) {
opts := options{
timingFilesPattern: dir.Join("*.log"),
pruneFilesMaxAgeDays: 2,
}

files, err := readAndPruneTimingReports(opts)
assert.NilError(t, err)
defer closeFiles(files)
assert.Equal(t, len(files), 2)

for _, fh := range files {
// check the files are properly seeked to 0
event, err := parseEvent(fh)
assert.NilError(t, err)
assert.Equal(t, event.Package, "pkg0")
}

actual, err := os.ReadDir(dir.Path())
assert.NilError(t, err)
assert.Equal(t, len(actual), 2)
})
}

func TestRun(t *testing.T) {
Expand Down Expand Up @@ -279,42 +256,3 @@ func formatJSON(t *testing.T, v io.Reader) string {
assert.NilError(t, err)
return string(formatted)
}

func TestRun_PruneOnly(t *testing.T) {
events := func(t *testing.T, start time.Time) string {
t.Helper()
var buf bytes.Buffer
encoder := json.NewEncoder(&buf)
for _, i := range []int{0, 1, 2} {
assert.NilError(t, encoder.Encode(testjson.TestEvent{
Time: start.Add(time.Duration(i) * time.Second),
Action: testjson.ActionRun,
Package: "pkg" + strconv.Itoa(i),
}))
}
return buf.String()
}

now := time.Now()
dir := fs.NewDir(t, "timing-files",
fs.WithFile("report1.log", events(t, now.Add(-time.Hour))),
fs.WithFile("report2.log", events(t, now.Add(-47*time.Hour))),
fs.WithFile("report3.log", events(t, now.Add(-49*time.Hour))),
fs.WithFile("report4.log", events(t, now.Add(-101*time.Hour))))

stdout := new(bytes.Buffer)
opts := options{
numPartitions: 3,
timingFilesPattern: dir.Join("*.log"),
debug: true,
stdout: stdout,
stdin: strings.NewReader(""),
pruneFilesMaxAgeDays: 2,
}
err := run(opts)
assert.NilError(t, err)

assert.Assert(t, fs.Equal(dir.Path(), fs.Expected(t,
fs.WithFile("report1.log", events(t, now.Add(-time.Hour))),
fs.WithFile("report2.log", events(t, now.Add(-47*time.Hour))))))
}

0 comments on commit d0adc24

Please sign in to comment.