diff --git a/bench_test.go b/bench_test.go new file mode 100644 index 000000000..f792ec921 --- /dev/null +++ b/bench_test.go @@ -0,0 +1,324 @@ +// MIT License +// +// Copyright (c) 2020 Plamen Petrov +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package main + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strconv" + "sync" + "testing" + + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + "github.com/ustiugov/fccd-orchestrator/metrics" +) + +const ( + benchDir = "bench_results" +) + +func TestBenchmarkServeWithCache(t *testing.T) { + var ( + servedTh uint64 + pinnedFuncNum int + isSyncOffload bool = true + ) + + images := getAllImages() + benchCount := 10 + vmID := 0 + + funcPool = NewFuncPool(!isSaveMemoryConst, servedTh, pinnedFuncNum, isTestModeConst) + + createResultsDir() + + for funcName, imageName := range images { + vmIDString := strconv.Itoa(vmID) + serveStats := make([]*metrics.Metric, benchCount) + + // Pull image + resp, _, err := funcPool.Serve(context.Background(), vmIDString, imageName, "replay") + require.NoError(t, err, "Function returned error") + require.Equal(t, resp.Payload, "Hello, replay_response!") + + message, err := funcPool.RemoveInstance(vmIDString, imageName, isSyncOffload) + require.NoError(t, err, "Function returned error, "+message) + // ----------------------------------------------------------------------- + + // Warm up loadsnapshot + resp, _, err = funcPool.Serve(context.Background(), vmIDString, imageName, "replay") + require.NoError(t, err, "Function returned error") + require.Equal(t, resp.Payload, "Hello, replay_response!") + + message, err = funcPool.RemoveInstance(vmIDString, imageName, isSyncOffload) + require.NoError(t, err, "Function returned error, "+message) + + for i := 0; i < benchCount; i++ { + resp, stat, err := funcPool.Serve(context.Background(), vmIDString, imageName, "replay") + require.NoError(t, err, "Function returned error") + require.Equal(t, resp.Payload, "Hello, replay_response!") + + serveStats[i] = stat + + message, err := funcPool.RemoveInstance(vmIDString, imageName, isSyncOffload) + require.NoError(t, err, "Function returned error, "+message) + } + + vmID++ + + outFileName := "serve_" + funcName + "_cache.txt" + metrics.PrintMeanStd(getOutFile(outFileName), serveStats...) + } +} + +func TestBenchmarkServeNoCache(t *testing.T) { + var ( + servedTh uint64 + pinnedFuncNum int + isSyncOffload bool = true + ) + + images := getAllImages() + benchCount := 10 + vmID := 10 + + funcPool = NewFuncPool(!isSaveMemoryConst, servedTh, pinnedFuncNum, isTestModeConst) + + createResultsDir() + + for funcName, imageName := range images { + vmIDString := strconv.Itoa(vmID) + serveStats := make([]*metrics.Metric, benchCount) + + // Pull image + resp, _, err := funcPool.Serve(context.Background(), vmIDString, imageName, "replay") + require.NoError(t, err, "Function returned error") + require.Equal(t, resp.Payload, "Hello, replay_response!") + + message, err := funcPool.RemoveInstance(vmIDString, imageName, isSyncOffload) + require.NoError(t, err, "Function returned error, "+message) + // ----------------------------------------------------------------------- + + // Warm up loadsnapshot + resp, _, err = funcPool.Serve(context.Background(), vmIDString, imageName, "replay") + require.NoError(t, err, "Function returned error") + require.Equal(t, resp.Payload, "Hello, replay_response!") + + message, err = funcPool.RemoveInstance(vmIDString, imageName, isSyncOffload) + require.NoError(t, err, "Function returned error, "+message) + + for i := 0; i < benchCount; i++ { + dropPageCache() + + resp, stat, err := funcPool.Serve(context.Background(), vmIDString, imageName, "replay") + require.NoError(t, err, "Function returned error") + require.Equal(t, resp.Payload, "Hello, replay_response!") + + serveStats[i] = stat + + message, err := funcPool.RemoveInstance(vmIDString, imageName, isSyncOffload) + require.NoError(t, err, "Function returned error, "+message) + } + + vmID++ + + outFileName := "serve_" + funcName + "_nocache.txt" + metrics.PrintMeanStd(getOutFile(outFileName), serveStats...) + } +} + +func dropPageCache() { + cmd := exec.Command("sudo", "/bin/sh", "-c", "sync; echo 1 > /proc/sys/vm/drop_caches") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stdout + + if err := cmd.Run(); err != nil { + log.Fatalf("Failed to drop caches: %v", err) + } +} + +func createResultsDir() { + if err := os.MkdirAll(benchDir, 0666); err != nil { + log.Fatalf("Failed to create results dir: %v", err) + } +} + +func getOutFile(name string) string { + return filepath.Join(benchDir, name) +} + +func getAllImages() map[string]string { + return map[string]string{ + "helloworld": "ustiugov/helloworld:var_workload", + "chameleon": "ustiugov/chameleon:var_workload", + "pyaes": "ustiugov/pyaes:var_workload", + "image_rotate": "ustiugov/image_rotate:var_workload", + "json_serdes": "ustiugov/json_serdes:var_workload", + //"lr_serving" : "ustiugov/lr_serving:var_workload", Issue#15 + //"cnn_serving" "ustiugov/cnn_serving:var_workload", + "rnn_serving": "ustiugov/rnn_serving:var_workload", + //"lr_training" : "ustiugov/lr_training:var_workload", + } +} + +func TestBenchParallelServeWithCache(t *testing.T) { + var ( + servedTh uint64 + pinnedFuncNum int + isSyncOffload bool = true + ) + + images := getAllImages() + parallel := 4 + vmID := 0 + + funcPool = NewFuncPool(!isSaveMemoryConst, servedTh, pinnedFuncNum, isTestModeConst) + + createResultsDir() + + for funcName, imageName := range images { + + serveMetrics := make([]*metrics.Metric, parallel) + + for i := 0; i < parallel; i++ { + vmIDString := strconv.Itoa(vmID + i) + + // Pull image and create VM + resp, _, err := funcPool.Serve(context.Background(), vmIDString, imageName, "replay") + require.NoError(t, err, "Function returned error") + require.Equal(t, resp.Payload, "Hello, replay_response!") + + message, err := funcPool.RemoveInstance(vmIDString, imageName, isSyncOffload) + require.NoError(t, err, "Function returned error, "+message) + // ----------------------------------------------------------------------- + + // Warm up loadsnapshot + resp, _, err = funcPool.Serve(context.Background(), vmIDString, imageName, "replay") + require.NoError(t, err, "Function returned error") + require.Equal(t, resp.Payload, "Hello, replay_response!") + + message, err = funcPool.RemoveInstance(vmIDString, imageName, isSyncOffload) + require.NoError(t, err, "Function returned error, "+message) + } + + var vmGroup sync.WaitGroup + start := make(chan struct{}) + + for i := 0; i < parallel; i++ { + vmIDString := strconv.Itoa(vmID + i) + + vmGroup.Add(1) + + go func(i int) { + <-start + defer vmGroup.Done() + + resp, metric, err := funcPool.Serve(context.Background(), vmIDString, imageName, "replay") + require.NoError(t, err, "Function returned error") + require.Equal(t, resp.Payload, "Hello, replay_response!") + + serveMetrics[i] = metric + + message, err := funcPool.RemoveInstance(vmIDString, imageName, isSyncOffload) + require.NoError(t, err, "Function returned error, "+message) + }(i) + } + + close(start) + vmGroup.Wait() + + vmID += parallel + + outFileName := "serve_" + funcName + "_par_cache.txt" + metrics.PrintMeanStd(getOutFile(outFileName), serveMetrics...) + } +} + +func TestBenchParallelServeNoCache(t *testing.T) { + var ( + servedTh uint64 + pinnedFuncNum int + isSyncOffload bool = true + ) + + images := getAllImages() + parallel := 4 + vmID := 0 + + funcPool = NewFuncPool(!isSaveMemoryConst, servedTh, pinnedFuncNum, isTestModeConst) + + createResultsDir() + + for funcName, imageName := range images { + + serveMetrics := make([]*metrics.Metric, parallel) + + for i := 0; i < parallel; i++ { + vmIDString := strconv.Itoa(vmID + i) + + // Pull image and create VM + resp, _, err := funcPool.Serve(context.Background(), vmIDString, imageName, "replay") + require.NoError(t, err, "Function returned error") + require.Equal(t, resp.Payload, "Hello, replay_response!") + + message, err := funcPool.RemoveInstance(vmIDString, imageName, isSyncOffload) + require.NoError(t, err, "Function returned error, "+message) + } + + var vmGroup sync.WaitGroup + start := make(chan struct{}) + + dropPageCache() + + for i := 0; i < parallel; i++ { + vmIDString := strconv.Itoa(vmID + i) + + vmGroup.Add(1) + + go func(i int) { + <-start + defer vmGroup.Done() + + resp, metric, err := funcPool.Serve(context.Background(), vmIDString, imageName, "replay") + require.NoError(t, err, "Function returned error") + require.Equal(t, resp.Payload, "Hello, replay_response!") + + serveMetrics[i] = metric + + message, err := funcPool.RemoveInstance(vmIDString, imageName, isSyncOffload) + require.NoError(t, err, "Function returned error, "+message) + }(i) + } + + close(start) + vmGroup.Wait() + + vmID += parallel + + outFileName := "serve_" + funcName + "_par_nocache.txt" + metrics.PrintMeanStd(getOutFile(outFileName), serveMetrics...) + } +} diff --git a/ctriface/Makefile b/ctriface/Makefile index c87ef0c28..4a9c66436 100644 --- a/ctriface/Makefile +++ b/ctriface/Makefile @@ -1,6 +1,8 @@ EXTRAGOARGS:=-v -race -cover EXTRATESTFILES:=iface_test.go iface.go dialer.go orch_options.go MANUALTESTFILES:=manual_cleanup_test.go iface.go dialer.go orch_options.go +BENCHFILES:=bench_test.go iface.go dialer.go orch_options.go +GOBENCH:=-v -timeout 1500s test: sudo env "PATH=$(PATH)" go test $(EXTRATESTFILES) $(EXTRAGOARGS) @@ -9,4 +11,7 @@ test-man: sudo env "PATH=$(PATH)" go test $(MANUALTESTFILES) $(EXTRAGOARGS) ./../scripts/clean_fcctr.sh -.PHONY: test test-man +bench: + sudo env "PATH=$(PATH)" go test $(BENCHFILES) $(GOBENCH) + ./../scripts/clean_fcctr.sh +.PHONY: test test-man bench diff --git a/ctriface/bench_test.go b/ctriface/bench_test.go new file mode 100644 index 000000000..c57a4d680 --- /dev/null +++ b/ctriface/bench_test.go @@ -0,0 +1,298 @@ +// MIT License +// +// Copyright (c) 2020 Plamen Petrov +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package ctriface + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strconv" + "testing" + "time" + + ctrdlog "github.com/containerd/containerd/log" + "github.com/containerd/containerd/namespaces" + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + "github.com/ustiugov/fccd-orchestrator/metrics" +) + +const ( + benchDir = "bench_results" +) + +func TestBenchmarkStart(t *testing.T) { + log.SetFormatter(&log.TextFormatter{ + TimestampFormat: ctrdlog.RFC3339NanoFixed, + FullTimestamp: true, + }) + //log.SetReportCaller(true) // FIXME: make sure it's false unless debugging + + log.SetOutput(os.Stdout) + + log.SetLevel(log.InfoLevel) + + testTimeout := 2000 * time.Second + ctx, cancel := context.WithTimeout(namespaces.WithNamespace(context.Background(), namespaceName), testTimeout) + defer cancel() + + orch := NewOrchestrator("devmapper", 1, WithTestModeOn(true)) + + images := getAllImages() + benchCount := 10 + vmID := 0 + + createResultsDir() + + for funcName, imageName := range images { + vmIDString := strconv.Itoa(vmID) + startMetrics := make([]*metrics.Metric, benchCount) + + // Pull image + message, _, err := orch.StartVM(ctx, vmIDString, imageName) + require.NoError(t, err, "Failed to start VM, "+message) + + message, err = orch.StopSingleVM(ctx, vmIDString) + require.NoError(t, err, "Failed to stop VM, "+message) + + for i := 0; i < benchCount; i++ { + message, metric, err := orch.StartVM(ctx, vmIDString, imageName) + require.NoError(t, err, "Failed to start VM, "+message) + startMetrics[i] = metric + + message, err = orch.StopSingleVM(ctx, vmIDString) + require.NoError(t, err, "Failed to stop VM, "+message) + } + + outFileName := "start_" + funcName + ".txt" + metrics.PrintMeanStd(getOutFile(outFileName), startMetrics...) + + vmID++ + + } + + orch.Cleanup() +} + +func TestBenchmarkLoadResumeWithCache(t *testing.T) { + log.SetFormatter(&log.TextFormatter{ + TimestampFormat: ctrdlog.RFC3339NanoFixed, + FullTimestamp: true, + }) + //log.SetReportCaller(true) // FIXME: make sure it's false unless debugging + + log.SetOutput(os.Stdout) + + log.SetLevel(log.InfoLevel) + + testTimeout := 2000 * time.Second + ctx, cancel := context.WithTimeout(namespaces.WithNamespace(context.Background(), namespaceName), testTimeout) + defer cancel() + + orch := NewOrchestrator("devmapper", 1, WithTestModeOn(true)) + + images := getAllImages() + benchCount := 10 + vmID := 0 + + createResultsDir() + + for funcName, imageName := range images { + vmIDString := strconv.Itoa(vmID) + loadMetrics := make([]*metrics.Metric, benchCount) + resumeMetrics := make([]*metrics.Metric, benchCount) + + // Pull image and prepare snapshot + message, _, err := orch.StartVM(ctx, vmIDString, imageName) + require.NoError(t, err, "Failed to start VM, "+message) + + message, err = orch.PauseVM(ctx, vmIDString) + require.NoError(t, err, "Failed to pause VM, "+vmIDString+", "+message) + + message, err = orch.CreateSnapshot(ctx, vmIDString) + require.NoError(t, err, "Failed to create snapshot of VM, "+message) + + message, err = orch.Offload(ctx, vmIDString) + require.NoError(t, err, "Failed to offload VM, "+message) + + time.Sleep(300 * time.Millisecond) + + message, _, err = orch.LoadSnapshot(ctx, vmIDString) + require.NoError(t, err, "Failed to load snapshot of VM, "+message) + + message, _, err = orch.ResumeVM(ctx, vmIDString) + require.NoError(t, err, "Failed to resume VM, "+message) + + message, err = orch.Offload(ctx, vmIDString) + require.NoError(t, err, "Failed to offload VM, "+message) + + time.Sleep(300 * time.Millisecond) + + for i := 0; i < benchCount; i++ { + message, loadMetric, err := orch.LoadSnapshot(ctx, vmIDString) + require.NoError(t, err, "Failed to load snapshot of VM, "+message) + + message, resumeMetric, err := orch.ResumeVM(ctx, vmIDString) + require.NoError(t, err, "Failed to resume VM, "+message) + + loadMetrics[i] = loadMetric + resumeMetrics[i] = resumeMetric + + message, err = orch.Offload(ctx, vmIDString) + require.NoError(t, err, "Failed to offload VM, "+message) + + time.Sleep(300 * time.Millisecond) + } + + outFileName := "load_" + funcName + "_cache.txt" + metrics.PrintMeanStd(getOutFile(outFileName), loadMetrics...) + + outFileName = "resume_" + funcName + "_cache.txt" + metrics.PrintMeanStd(getOutFile(outFileName), resumeMetrics...) + + vmID++ + + } + + orch.Cleanup() +} + +func TestBenchmarkLoadResumeNoCache(t *testing.T) { + log.SetFormatter(&log.TextFormatter{ + TimestampFormat: ctrdlog.RFC3339NanoFixed, + FullTimestamp: true, + }) + //log.SetReportCaller(true) // FIXME: make sure it's false unless debugging + + log.SetOutput(os.Stdout) + + log.SetLevel(log.InfoLevel) + + testTimeout := 2000 * time.Second + ctx, cancel := context.WithTimeout(namespaces.WithNamespace(context.Background(), namespaceName), testTimeout) + defer cancel() + + orch := NewOrchestrator("devmapper", 1, WithTestModeOn(true)) + + images := getAllImages() + benchCount := 10 + vmID := 20 + + createResultsDir() + + for funcName, imageName := range images { + vmIDString := strconv.Itoa(vmID) + loadMetrics := make([]*metrics.Metric, benchCount) + resumeMetrics := make([]*metrics.Metric, benchCount) + + // Pull image and prepare snapshot + message, _, err := orch.StartVM(ctx, vmIDString, imageName) + require.NoError(t, err, "Failed to start VM, "+message) + + message, err = orch.PauseVM(ctx, vmIDString) + require.NoError(t, err, "Failed to pause VM, "+vmIDString+", "+message) + + message, err = orch.CreateSnapshot(ctx, vmIDString) + require.NoError(t, err, "Failed to create snapshot of VM, "+message) + + message, err = orch.Offload(ctx, vmIDString) + require.NoError(t, err, "Failed to offload VM, "+message) + + time.Sleep(300 * time.Millisecond) + + message, _, err = orch.LoadSnapshot(ctx, vmIDString) + require.NoError(t, err, "Failed to load snapshot of VM, "+message) + + message, _, err = orch.ResumeVM(ctx, vmIDString) + require.NoError(t, err, "Failed to resume VM, "+message) + + message, err = orch.Offload(ctx, vmIDString) + require.NoError(t, err, "Failed to offload VM, "+message) + + time.Sleep(300 * time.Millisecond) + + for i := 0; i < benchCount; i++ { + dropPageCache() + + message, loadMetric, err := orch.LoadSnapshot(ctx, vmIDString) + require.NoError(t, err, "Failed to load snapshot of VM, "+message) + + message, resumeMetric, err := orch.ResumeVM(ctx, vmIDString) + require.NoError(t, err, "Failed to resume VM, "+message) + + loadMetrics[i] = loadMetric + resumeMetrics[i] = resumeMetric + + message, err = orch.Offload(ctx, vmIDString) + require.NoError(t, err, "Failed to offload VM, "+message) + + time.Sleep(300 * time.Millisecond) + } + + outFileName := "load_" + funcName + "_nocache.txt" + metrics.PrintMeanStd(getOutFile(outFileName), loadMetrics...) + + outFileName = "resume_" + funcName + "_nocache.txt" + metrics.PrintMeanStd(getOutFile(outFileName), resumeMetrics...) + + vmID++ + + } + + orch.Cleanup() +} + +func dropPageCache() { + cmd := exec.Command("sudo", "/bin/sh", "-c", "sync; echo 1 > /proc/sys/vm/drop_caches") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stdout + + if err := cmd.Run(); err != nil { + log.Fatalf("Failed to drop caches: %v", err) + } +} + +func createResultsDir() { + if err := os.MkdirAll(benchDir, 0666); err != nil { + log.Fatalf("Failed to create results dir: %v", err) + } +} + +func getOutFile(name string) string { + return filepath.Join(benchDir, name) +} + +func getAllImages() map[string]string { + return map[string]string{ + "helloworld": "ustiugov/helloworld:var_workload", + "chameleon": "ustiugov/chameleon:var_workload", + "pyaes": "ustiugov/pyaes:var_workload", + "image_rotate": "ustiugov/image_rotate:var_workload", + "json_serdes": "ustiugov/json_serdes:var_workload", + //"lr_serving" : "ustiugov/lr_serving:var_workload", Issue#15 + //"cnn_serving" "ustiugov/cnn_serving:var_workload", + "rnn_serving": "ustiugov/rnn_serving:var_workload", + //"lr_training" : "ustiugov/lr_training:var_workload", + } +} diff --git a/ctriface/failing_test.go b/ctriface/failing_test.go index c5d1872b8..7f4d2532d 100644 --- a/ctriface/failing_test.go +++ b/ctriface/failing_test.go @@ -38,7 +38,7 @@ func TestStartSnapStop(t *testing.T) { message, err = orch.PauseVM(ctx, vmID) require.NoError(t, err, "Failed to pause VM, "+message) - message, err = orch.CreateSnapshot(ctx, vmID, "/tmp/snapshot_file", "/tmp/mem_file") + message, err = orch.CreateSnapshot(ctx, vmID) require.NoError(t, err, "Failed to create snapshot of VM, "+message) message, err = orch.Offload(ctx, vmID) @@ -46,10 +46,10 @@ func TestStartSnapStop(t *testing.T) { time.Sleep(300 * time.Millisecond) - message, err = orch.LoadSnapshot(ctx, vmID, "/tmp/snapshot_file", "/tmp/mem_file") + message, _, err = orch.LoadSnapshot(ctx, vmID) require.NoError(t, err, "Failed to load snapshot of VM, "+message) - message, err = orch.ResumeVM(ctx, vmID) + message, _, err = orch.ResumeVM(ctx, vmID) require.NoError(t, err, "Failed to resume VM, "+message) message, err = orch.StopSingleVMOnly(ctx, vmID) diff --git a/ctriface/go.mod b/ctriface/go.mod index 4f045a55f..13c2727bd 100644 --- a/ctriface/go.mod +++ b/ctriface/go.mod @@ -12,11 +12,11 @@ require ( github.com/firecracker-microvm/firecracker-containerd v0.0.0-00010101000000-000000000000 github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.6.0 - github.com/stretchr/testify v1.5.1 + github.com/stretchr/testify v1.6.1 github.com/ustiugov/fccd-orchestrator/helloworld v0.0.0-20200717125634-528c6e9f9cc9 + github.com/ustiugov/fccd-orchestrator/metrics v0.0.0-20200727115924-ea27568b4c04 github.com/ustiugov/fccd-orchestrator/misc v0.0.0-20200717125634-528c6e9f9cc9 github.com/ustiugov/fccd-orchestrator/taps v0.0.0-20200717125634-528c6e9f9cc9 - golang.org/x/tools v0.0.0-20200721223218-6123e77877b2 // indirect google.golang.org/grpc v1.30.0 ) diff --git a/ctriface/go.sum b/ctriface/go.sum index 1ceb52793..f88265711 100644 --- a/ctriface/go.sum +++ b/ctriface/go.sum @@ -19,6 +19,7 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdko github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705 h1:UUppSQnhf4Yc6xGxSkoQpPhb7RVzuv5Nb1mwJ5VId9s= github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= @@ -127,6 +128,7 @@ github.com/firecracker-microvm/firecracker-containerd v0.0.0-20200613001316-5b51 github.com/firecracker-microvm/firecracker-go-sdk v0.20.1-0.20200204230548-c56932849923/go.mod h1:kW0gxvPpPvMukUxxTO9DrpSlScrtrTDGY3VgjAj/Qwc= github.com/firecracker-microvm/firecracker-go-sdk v0.21.0 h1:41W/zyL3S33ZK/pNFfVrW38Y2YpNE6ZFArQOHL27V2I= github.com/firecracker-microvm/firecracker-go-sdk v0.21.0/go.mod h1:zyc9BrKGePpNLbQ5y2ZtdzXEfpMJeHPeFNVpyo0S1WQ= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= @@ -215,6 +217,7 @@ github.com/gogo/protobuf v1.3.0 h1:G8O7TerXerS4F6sx9OV7/nRfJdnXgHZu/S/7F2SN+UE= github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -255,6 +258,7 @@ github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6t github.com/juju/errors v0.0.0-20180806074554-22422dad46e1/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= github.com/juju/testing v0.0.0-20190613124551-e81189438503/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -347,6 +351,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 h1:b6uOv7YOFK0TYG7HtkIgExQo+2RdLuwRft63jn2HWj8= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= @@ -369,6 +375,8 @@ github.com/ustiugov/fccd-orchestrator v0.0.0-20200714114243-04230dc2a517 h1:Rne2 github.com/ustiugov/fccd-orchestrator v0.0.0-20200714121730-5e9ab120e967 h1:32Fnn51pqXR+ApgBFGaVqegcQx5a3XqmtCBrGLNmi0s= github.com/ustiugov/fccd-orchestrator v0.0.0-20200714125803-ec58429be14d h1:UW5RhvDszUgAIa1UVwlxiDLk1NfHCA4mllE8YrqYcYc= github.com/ustiugov/fccd-orchestrator v0.0.0-20200714130355-0a4bf3145bf7 h1:evWpQ4LCzOfDQmgXEiLg6ldTpAiwbQlK3ky/u5TTT3A= +github.com/ustiugov/fccd-orchestrator v0.0.0-20200722115050-c92a0ba639e9 h1:bl70tn2A2ir7opdglOZKfi2hrnFQXiavWKNUM361MHw= +github.com/ustiugov/fccd-orchestrator v0.0.0-20200722141002-55dbdeb43861 h1:NKd5TlZCGrL59LDX7oTX9zF9isTUfgW2NKW1bFDiRIg= github.com/ustiugov/fccd-orchestrator/ctriface v0.0.0-20200416161917-c9cd3cf6dbcf/go.mod h1:2x1L5ydHhOsleqG/GQ9MLWgjF1YQF91u5pe0TvKEjhI= github.com/ustiugov/fccd-orchestrator/helloworld v0.0.0-20200710144657-9fbec6857e48 h1:BXnRBeRgnVfl4oYD9720hsSe5TScGSP8bpVBBpHEQMw= github.com/ustiugov/fccd-orchestrator/helloworld v0.0.0-20200710144657-9fbec6857e48/go.mod h1:5dhCs/XynpQoQcrhd/YgUBjGahhNpTknQUcC1kHRCaA= @@ -384,6 +392,18 @@ github.com/ustiugov/fccd-orchestrator/helloworld v0.0.0-20200714130355-0a4bf3145 github.com/ustiugov/fccd-orchestrator/helloworld v0.0.0-20200714130355-0a4bf3145bf7/go.mod h1:5dhCs/XynpQoQcrhd/YgUBjGahhNpTknQUcC1kHRCaA= github.com/ustiugov/fccd-orchestrator/helloworld v0.0.0-20200717125634-528c6e9f9cc9 h1:1EQvsISd3xbC1krrXCCuIsingP9PF4M6pfCTXPnhYb4= github.com/ustiugov/fccd-orchestrator/helloworld v0.0.0-20200717125634-528c6e9f9cc9/go.mod h1:5dhCs/XynpQoQcrhd/YgUBjGahhNpTknQUcC1kHRCaA= +github.com/ustiugov/fccd-orchestrator/metrics v0.0.0-20200722141002-55dbdeb43861 h1:rC47RJSLGILLDcXJ87gPnDZ/3DlfYcSWnAge3z8kQq8= +github.com/ustiugov/fccd-orchestrator/metrics v0.0.0-20200722141002-55dbdeb43861/go.mod h1:veO20AvWGKpDk3iD0IWJpfEnvF8PJVMK38n3SOWKKR0= +github.com/ustiugov/fccd-orchestrator/metrics v0.0.0-20200723085111-a9d697412510 h1:2F3Bp/P7C2UMKU+vakjEW/++E2yCntucHKNiRxcxQOE= +github.com/ustiugov/fccd-orchestrator/metrics v0.0.0-20200723085111-a9d697412510/go.mod h1:NOrYF6usVkdr6gpXAP7pk7jp1MlAXlrZQuUEOVhRuV8= +github.com/ustiugov/fccd-orchestrator/metrics v0.0.0-20200724084859-4128c20228f6 h1:1zUVOTn8ivWzApfn1hHrIavvqSJe8rF05j3goNIcvg4= +github.com/ustiugov/fccd-orchestrator/metrics v0.0.0-20200724084859-4128c20228f6/go.mod h1:NOrYF6usVkdr6gpXAP7pk7jp1MlAXlrZQuUEOVhRuV8= +github.com/ustiugov/fccd-orchestrator/metrics v0.0.0-20200727091444-0c206f69e5c4 h1:104hqCZ20UoalkNMrykJAD1VMCpIghMGF12WX8mgSlA= +github.com/ustiugov/fccd-orchestrator/metrics v0.0.0-20200727091444-0c206f69e5c4/go.mod h1:NOrYF6usVkdr6gpXAP7pk7jp1MlAXlrZQuUEOVhRuV8= +github.com/ustiugov/fccd-orchestrator/metrics v0.0.0-20200727100916-b838e9b56a67 h1:KHuietztRcEBIIntWQbj4a8O4gqkhl8hDpPdE92O3fo= +github.com/ustiugov/fccd-orchestrator/metrics v0.0.0-20200727100916-b838e9b56a67/go.mod h1:NOrYF6usVkdr6gpXAP7pk7jp1MlAXlrZQuUEOVhRuV8= +github.com/ustiugov/fccd-orchestrator/metrics v0.0.0-20200727115924-ea27568b4c04 h1:H5PUKmgr+tRmq48OPnfVJfFoSwD7YC1tYIhaT2MjRDQ= +github.com/ustiugov/fccd-orchestrator/metrics v0.0.0-20200727115924-ea27568b4c04/go.mod h1:kqi59gkrrgkSH4W9PH85SL00/kLZysF4NlmmpZFLs58= github.com/ustiugov/fccd-orchestrator/misc v0.0.0-20200416161510-f6f32fa5d52b h1:HAh2FBZ0QkBGxU/llIVVOhfnqCPEkE353FpZg7Y6sh0= github.com/ustiugov/fccd-orchestrator/misc v0.0.0-20200416161510-f6f32fa5d52b/go.mod h1:tPZXBU39WgZYegI7Cxm6k2434V9WR/4xyuTBK0MrlBw= github.com/ustiugov/fccd-orchestrator/misc v0.0.0-20200416171230-6533745bd3b1 h1:5BWRtXybC0fFszNmZHqWzPaNNZ0ctLfWvYey0DEKjAk= @@ -496,7 +516,11 @@ golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56 h1:ZpKuNIejY8P0ExLOVyKhb0 golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -571,11 +595,13 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -591,10 +617,17 @@ golang.org/x/tools v0.0.0-20200721032237-77f530d86f9a h1:kVMPw4f6EVqYdfGQTedjrpw golang.org/x/tools v0.0.0-20200721032237-77f530d86f9a/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200721223218-6123e77877b2 h1:kxDWg8KNMtpGjI/XVKGgOtSljTnVg/PrjhS8+0pxjLE= golang.org/x/tools v0.0.0-20200721223218-6123e77877b2/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200723000907-a7c6fd066f6d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200725200936-102e7d357031/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.7.0 h1:Hdks0L0hgznZLG9nzXb8vZ0rRvqNvAcgAp84y7Mwkgw= +gonum.org/v1/gonum v0.7.0/go.mod h1:L02bwd0sqlsvRv41G7wGWFCsVNZFv/k1xzGIxeANHGM= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -644,6 +677,8 @@ gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -651,3 +686,4 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= istio.io/gogo-genproto v0.0.0-20190731221249-06e20ada0df2/go.mod h1:IjvrbUlRbbw4JCpsgvgihcz9USUwEoNTL/uwMtyV5yk= k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/ctriface/iface.go b/ctriface/iface.go index cdac56afd..3ff649795 100644 --- a/ctriface/iface.go +++ b/ctriface/iface.go @@ -27,7 +27,7 @@ import ( "fmt" "os" "os/signal" - "strconv" + "path/filepath" "sync" "syscall" "time" @@ -51,6 +51,7 @@ import ( _ "google.golang.org/grpc/status" //tmp hpb "github.com/ustiugov/fccd-orchestrator/helloworld" + "github.com/ustiugov/fccd-orchestrator/metrics" "github.com/ustiugov/fccd-orchestrator/misc" _ "github.com/davecgh/go-spew/spew" //tmp @@ -73,6 +74,7 @@ type Orchestrator struct { // store *skv.KVStore snapshotsEnabled bool isUPFEnabled bool + snapshotsDir string } // NewOrchestrator Initializes a new orchestrator @@ -84,11 +86,22 @@ func NewOrchestrator(snapshotter string, niNum int, opts ...OrchestratorOption) o.vmPool = misc.NewVMPool(o.niNum) o.cachedImages = make(map[string]containerd.Image) o.snapshotter = snapshotter + o.snapshotsDir = "/fccd/snapshots" for _, opt := range opts { opt(o) } + if _, err := os.Stat(o.snapshotsDir); err != nil { + if !os.IsNotExist(err) { + log.Panicf("Snapshot dir %s exists", o.snapshotsDir) + } + } + + if err := os.MkdirAll(o.snapshotsDir, 0666); err != nil { + log.Panicf("Failed to create snapshots dir %s", o.snapshotsDir) + } + log.Info("Creating containerd client") o.client, err = containerd.New(containerdAddress) if err != nil { @@ -177,9 +190,12 @@ func (o *Orchestrator) getFuncClient(ctx context.Context, vm *misc.VM, logger *l } // StartVM Boots a VM if it does not exist -func (o *Orchestrator) StartVM(ctx context.Context, vmID, imageName string) (string, string, error) { - var tProfile string - var tStart, tElapsed time.Time +func (o *Orchestrator) StartVM(ctx context.Context, vmID, imageName string) (string, *metrics.Metric, error) { + var ( + startVMMetric *metrics.Metric = metrics.NewMetric() + tStart time.Time + ) + logger := log.WithFields(log.Fields{"vmID": vmID, "image": imageName}) logger.Debug("StartVM: Received StartVM") @@ -187,27 +203,25 @@ func (o *Orchestrator) StartVM(ctx context.Context, vmID, imageName string) (str vm, err := o.vmPool.Allocate(vmID) if err != nil { logger.Panic("StartVM: Unknown error") - return "StartVM: Unknown error", tProfile, err + return "StartVM: Unknown error", startVMMetric, err } ctx = namespaces.WithNamespace(ctx, namespaceName) tStart = time.Now() if vm.Image, err = o.getImage(ctx, imageName); err != nil { - return "Failed to start VM", tProfile, errors.Wrapf(err, "Failed to get/pull image") + return "Failed to start VM", startVMMetric, errors.Wrapf(err, "Failed to get/pull image") } - tElapsed = time.Now() - tProfile += strconv.FormatInt(tElapsed.Sub(tStart).Microseconds(), 10) + ";" + startVMMetric.MetricMap[metrics.GetImage] = metrics.ToUS(time.Since(tStart)) logger.Debug("StartVM: Creating a new VM") tStart = time.Now() _, err = o.fcClient.CreateVM(ctx, o.getVMConfig(vm)) - tElapsed = time.Now() - tProfile += strconv.FormatInt(tElapsed.Sub(tStart).Microseconds(), 10) + ";" + startVMMetric.MetricMap[metrics.FcCreateVM] = metrics.ToUS(time.Since(tStart)) if err != nil { if errCleanup := o.cleanup(ctx, vm, false, false, false, false); errCleanup != nil { logger.Warn("Cleanup failed: ", errCleanup) } - return "Failed to start VM", tProfile, errors.Wrap(err, "failed to create the VM") + return "Failed to start VM", startVMMetric, errors.Wrap(err, "failed to create the VM") } logger.Debug("StartVM: Creating a new container") @@ -224,40 +238,37 @@ func (o *Orchestrator) StartVM(ctx context.Context, vmID, imageName string) (str ), containerd.WithRuntime("aws.firecracker", nil), ) + startVMMetric.MetricMap[metrics.NewContainer] = metrics.ToUS(time.Since(tStart)) vm.Container = &container - tElapsed = time.Now() - tProfile += strconv.FormatInt(tElapsed.Sub(tStart).Microseconds(), 10) + ";" if err != nil { if errCleanup := o.cleanup(ctx, vm, true, false, false, false); errCleanup != nil { logger.Warn("Cleanup failed: ", errCleanup) } - return "Failed to start VM", tProfile, errors.Wrap(err, "failed to create a container") + return "Failed to start VM", startVMMetric, errors.Wrap(err, "failed to create a container") } logger.Debug("StartVM: Creating a new task") tStart = time.Now() task, err := container.NewTask(ctx, cio.NewCreator(cio.WithStdio)) + startVMMetric.MetricMap[metrics.NewTask] = metrics.ToUS(time.Since(tStart)) vm.Task = &task - tElapsed = time.Now() - tProfile += strconv.FormatInt(tElapsed.Sub(tStart).Microseconds(), 10) + ";" if err != nil { if errCleanup := o.cleanup(ctx, vm, true, true, false, false); errCleanup != nil { logger.Warn("Cleanup failed: ", errCleanup) } - return "Failed to start VM", tProfile, errors.Wrap(err, "failed to create a task") + return "Failed to start VM", startVMMetric, errors.Wrap(err, "failed to create a task") } logger.Debug("StartVM: Waiting for the task to get ready") tStart = time.Now() ch, err := task.Wait(ctx) + startVMMetric.MetricMap[metrics.TaskWait] = metrics.ToUS(time.Since(tStart)) vm.TaskCh = ch - tElapsed = time.Now() - tProfile += strconv.FormatInt(tElapsed.Sub(tStart).Microseconds(), 10) + ";" if err != nil { if errCleanup := o.cleanup(ctx, vm, true, true, true, false); errCleanup != nil { logger.Warn("Cleanup failed: ", errCleanup) } - return "Failed to start VM", tProfile, errors.Wrap(err, "failed to wait for a task") + return "Failed to start VM", startVMMetric, errors.Wrap(err, "failed to wait for a task") } logger.Debug("StartVM: Starting the task") @@ -266,20 +277,19 @@ func (o *Orchestrator) StartVM(ctx context.Context, vmID, imageName string) (str if errCleanup := o.cleanup(ctx, vm, true, true, true, false); errCleanup != nil { logger.Warn("Cleanup failed: ", errCleanup) } - return "Failed to start VM", tProfile, errors.Wrap(err, "failed to start a task") + return "Failed to start VM", startVMMetric, errors.Wrap(err, "failed to start a task") } - tElapsed = time.Now() - tProfile += strconv.FormatInt(tElapsed.Sub(tStart).Microseconds(), 10) + ";" + startVMMetric.MetricMap[metrics.TaskStart] = metrics.ToUS(time.Since(tStart)) funcClient, err := o.getFuncClient(ctx, vm, logger) if err != nil { - return "Failed to start VM", tProfile, errors.Wrap(err, "failed to connect to a function") + return "Failed to start VM", startVMMetric, errors.Wrap(err, "failed to connect to a function") } vm.FuncClient = &funcClient logger.Debug("Successfully started a VM") - return "VM, container, and task started successfully", tProfile, nil + return "VM, container, and task started successfully", startVMMetric, nil } // GetFuncClient Returns the client for the function @@ -437,8 +447,12 @@ func (o *Orchestrator) setupCloseHandler() { } // Cleanup Removes the bridges created by the VM pool's tap manager +// Cleans up snapshots directory func (o *Orchestrator) Cleanup() { o.vmPool.RemoveBridges() + if err := os.RemoveAll(o.snapshotsDir); err != nil { + log.Panic("failed to delete snapshots dir", err) + } } // GetSnapshotsEnabled Returns the snapshots mode of the orchestrator @@ -467,33 +481,47 @@ func (o *Orchestrator) PauseVM(ctx context.Context, vmID string) (string, error) } // ResumeVM Resumes a VM -func (o *Orchestrator) ResumeVM(ctx context.Context, vmID string) (string, error) { +func (o *Orchestrator) ResumeVM(ctx context.Context, vmID string) (string, *metrics.Metric, error) { + var ( + resumeVMMetric *metrics.Metric = metrics.NewMetric() + tStart time.Time + ) + logger := log.WithFields(log.Fields{"vmID": vmID}) logger.Debug("Orchestrator received ResumeVM") ctx = namespaces.WithNamespace(ctx, namespaceName) + tStart = time.Now() if _, err := o.fcClient.ResumeVM(ctx, &proto.ResumeVMRequest{VMID: vmID}); err != nil { logger.Warn("failed to pause the VM: ", err) - return "Resuming VM " + vmID + " failed", err + return "Resuming VM " + vmID + " failed", resumeVMMetric, err } + resumeVMMetric.MetricMap[metrics.FcResume] = metrics.ToUS(time.Since(tStart)) vm, err := o.vmPool.GetVM(vmID) if err != nil { - return "Snapshot of VM " + vmID + " loaded successfully", err + return "Snapshot of VM " + vmID + " loaded successfully", resumeVMMetric, err } + tStart = time.Now() funcClient, err := o.getFuncClient(ctx, vm, logger) if err != nil { - return "Failed to start VM", errors.Wrap(err, "failed to connect to a function") + return "Failed to start VM", resumeVMMetric, errors.Wrap(err, "failed to connect to a function") } vm.FuncClient = &funcClient + resumeVMMetric.MetricMap[metrics.ReconnectFuncClient] = metrics.ToUS(time.Since(tStart)) - return "VM " + vmID + " resumed successfully", nil + return "VM " + vmID + " resumed successfully", resumeVMMetric, nil } // CreateSnapshot Creates a snapshot of a VM -func (o *Orchestrator) CreateSnapshot(ctx context.Context, vmID, snapPath, memPath string) (string, error) { +func (o *Orchestrator) CreateSnapshot(ctx context.Context, vmID string) (string, error) { + var ( + snapPath string = filepath.Join(o.snapshotsDir, "snap_file_"+vmID) + memPath string = filepath.Join(o.snapshotsDir, "mem_file_"+vmID) + ) + logger := log.WithFields(log.Fields{"vmID": vmID}) logger.Debug("Orchestrator received CreateSnapshot") @@ -510,7 +538,14 @@ func (o *Orchestrator) CreateSnapshot(ctx context.Context, vmID, snapPath, memPa } // LoadSnapshot Loads a snapshot of a VM -func (o *Orchestrator) LoadSnapshot(ctx context.Context, vmID, snapPath, memPath string) (string, error) { +func (o *Orchestrator) LoadSnapshot(ctx context.Context, vmID string) (string, *metrics.Metric, error) { + var ( + loadSnapshotMetric *metrics.Metric = metrics.NewMetric() + tStart time.Time + snapPath string = filepath.Join(o.snapshotsDir, "snap_file_"+vmID) + memPath string = filepath.Join(o.snapshotsDir, "mem_file_"+vmID) + ) + logger := log.WithFields(log.Fields{"vmID": vmID}) logger.Debug("Orchestrator received LoadSnapshot") @@ -523,12 +558,14 @@ func (o *Orchestrator) LoadSnapshot(ctx context.Context, vmID, snapPath, memPath EnableUserPF: o.GetUPFEnabled(), } + tStart = time.Now() if _, err := o.fcClient.LoadSnapshot(ctx, req); err != nil { logger.Warn("failed to load snapshot of the VM: ", err) - return "Loading snapshot of VM " + vmID + " failed", err + return "Loading snapshot of VM " + vmID + " failed", loadSnapshotMetric, err } + loadSnapshotMetric.MetricMap[metrics.Full] = metrics.ToUS(time.Since(tStart)) - return "Snapshot of VM " + vmID + " loaded successfully", nil + return "Snapshot of VM " + vmID + " loaded successfully", loadSnapshotMetric, nil } // Offload Shuts down the VM but leaves shim and other resources running. diff --git a/ctriface/iface_test.go b/ctriface/iface_test.go index b695e173b..6348b3255 100644 --- a/ctriface/iface_test.go +++ b/ctriface/iface_test.go @@ -39,10 +39,10 @@ func TestPauseSnapResume(t *testing.T) { message, err = orch.PauseVM(ctx, vmID) require.NoError(t, err, "Failed to pause VM, "+message) - message, err = orch.CreateSnapshot(ctx, vmID, "/tmp/snapshot_file", "/tmp/mem_file") + message, err = orch.CreateSnapshot(ctx, vmID) require.NoError(t, err, "Failed to create snapshot of VM, "+message) - message, err = orch.ResumeVM(ctx, vmID) + message, _, err = orch.ResumeVM(ctx, vmID) require.NoError(t, err, "Failed to resume VM, "+message) message, err = orch.StopSingleVM(ctx, vmID) @@ -104,7 +104,7 @@ func TestPauseResumeSerial(t *testing.T) { message, err = orch.PauseVM(ctx, vmID) require.NoError(t, err, "Failed to pause VM, "+message) - message, err = orch.ResumeVM(ctx, vmID) + message, _, err = orch.ResumeVM(ctx, vmID) require.NoError(t, err, "Failed to resume VM, "+message) message, err = orch.StopSingleVM(ctx, vmID) @@ -215,7 +215,7 @@ func TestPauseResumeParallel(t *testing.T) { go func(i int) { defer vmGroup.Done() vmID := fmt.Sprintf("%d", i) - message, err := orch.ResumeVM(ctx, vmID) + message, _, err := orch.ResumeVM(ctx, vmID) require.NoError(t, err, "Failed to resume VM, "+message) }(i) } diff --git a/ctriface/manual_cleanup_test.go b/ctriface/manual_cleanup_test.go index d56ebd141..4a196d7f9 100644 --- a/ctriface/manual_cleanup_test.go +++ b/ctriface/manual_cleanup_test.go @@ -41,10 +41,10 @@ func TestSnapLoad(t *testing.T) { message, err = orch.PauseVM(ctx, vmID) require.NoError(t, err, "Failed to pause VM, "+message) - message, err = orch.CreateSnapshot(ctx, vmID, "/tmp/snapshot_file", "/tmp/mem_file") + message, err = orch.CreateSnapshot(ctx, vmID) require.NoError(t, err, "Failed to create snapshot of VM, "+message) - message, err = orch.ResumeVM(ctx, vmID) + message, _, err = orch.ResumeVM(ctx, vmID) require.NoError(t, err, "Failed to resume VM, "+message) message, err = orch.Offload(ctx, vmID) @@ -52,10 +52,10 @@ func TestSnapLoad(t *testing.T) { time.Sleep(300 * time.Millisecond) - message, err = orch.LoadSnapshot(ctx, vmID, "/tmp/snapshot_file", "/tmp/mem_file") + message, _, err = orch.LoadSnapshot(ctx, vmID) require.NoError(t, err, "Failed to load snapshot of VM, "+message) - message, err = orch.ResumeVM(ctx, vmID) + message, _, err = orch.ResumeVM(ctx, vmID) require.NoError(t, err, "Failed to resume VM, "+message) orch.Cleanup() @@ -87,7 +87,7 @@ func TestSnapLoadMultiple(t *testing.T) { message, err = orch.PauseVM(ctx, vmID) require.NoError(t, err, "Failed to pause VM, "+message) - message, err = orch.CreateSnapshot(ctx, vmID, "/tmp/snapshot_file1", "/tmp/mem_file1") + message, err = orch.CreateSnapshot(ctx, vmID) require.NoError(t, err, "Failed to create snapshot of VM, "+message) message, err = orch.Offload(ctx, vmID) @@ -95,10 +95,10 @@ func TestSnapLoadMultiple(t *testing.T) { time.Sleep(300 * time.Millisecond) - message, err = orch.LoadSnapshot(ctx, vmID, "/tmp/snapshot_file1", "/tmp/mem_file1") + message, _, err = orch.LoadSnapshot(ctx, vmID) require.NoError(t, err, "Failed to load snapshot of VM, "+message) - message, err = orch.ResumeVM(ctx, vmID) + message, _, err = orch.ResumeVM(ctx, vmID) require.NoError(t, err, "Failed to resume VM, "+message) message, err = orch.Offload(ctx, vmID) @@ -106,10 +106,10 @@ func TestSnapLoadMultiple(t *testing.T) { time.Sleep(300 * time.Millisecond) - message, err = orch.LoadSnapshot(ctx, vmID, "/tmp/snapshot_file1", "/tmp/mem_file1") + message, _, err = orch.LoadSnapshot(ctx, vmID) require.NoError(t, err, "Failed to load snapshot of VM, "+message) - message, err = orch.ResumeVM(ctx, vmID) + message, _, err = orch.ResumeVM(ctx, vmID) require.NoError(t, err, "Failed to resume VM, "+message) message, err = orch.Offload(ctx, vmID) @@ -151,8 +151,6 @@ func TestParallelSnapLoad(t *testing.T) { go func(i int) { defer vmGroup.Done() vmID := fmt.Sprintf("%d", i+vmIDBase) - snapshotFilePath := fmt.Sprintf("/dev/snapshot_file_%s", vmID) - memFilePath := fmt.Sprintf("/dev/mem_file_%s", vmID) message, _, err := orch.StartVM(ctx, vmID, "ustiugov/helloworld:runner_workload") require.NoError(t, err, "Failed to start VM, "+vmID+", "+message) @@ -160,7 +158,7 @@ func TestParallelSnapLoad(t *testing.T) { message, err = orch.PauseVM(ctx, vmID) require.NoError(t, err, "Failed to pause VM, "+vmID+", "+message) - message, err = orch.CreateSnapshot(ctx, vmID, snapshotFilePath, memFilePath) + message, err = orch.CreateSnapshot(ctx, vmID) require.NoError(t, err, "Failed to create snapshot of VM, "+vmID+", "+message) message, err = orch.Offload(ctx, vmID) @@ -168,10 +166,10 @@ func TestParallelSnapLoad(t *testing.T) { time.Sleep(300 * time.Millisecond) - message, err = orch.LoadSnapshot(ctx, vmID, snapshotFilePath, memFilePath) + message, _, err = orch.LoadSnapshot(ctx, vmID) require.NoError(t, err, "Failed to load snapshot of VM, "+vmID+", "+message) - message, err = orch.ResumeVM(ctx, vmID) + message, _, err = orch.ResumeVM(ctx, vmID) require.NoError(t, err, "Failed to resume VM, "+vmID+", "+message) }(i) } @@ -242,9 +240,7 @@ func TestParallelPhasedSnapLoad(t *testing.T) { go func(i int) { defer vmGroup.Done() vmID := fmt.Sprintf("%d", i+vmIDBase) - snapshotFilePath := fmt.Sprintf("/dev/snapshot_file_%s", vmID) - memFilePath := fmt.Sprintf("/dev/mem_file_%s", vmID) - message, err := orch.CreateSnapshot(ctx, vmID, snapshotFilePath, memFilePath) + message, err := orch.CreateSnapshot(ctx, vmID) require.NoError(t, err, "Failed to create snapshot of VM, "+vmID+", "+message) }(i) } @@ -274,9 +270,7 @@ func TestParallelPhasedSnapLoad(t *testing.T) { go func(i int) { defer vmGroup.Done() vmID := fmt.Sprintf("%d", i+vmIDBase) - snapshotFilePath := fmt.Sprintf("/dev/snapshot_file_%s", vmID) - memFilePath := fmt.Sprintf("/dev/mem_file_%s", vmID) - message, err := orch.LoadSnapshot(ctx, vmID, snapshotFilePath, memFilePath) + message, _, err := orch.LoadSnapshot(ctx, vmID) require.NoError(t, err, "Failed to load snapshot of VM, "+vmID+", "+message) }(i) } @@ -290,7 +284,7 @@ func TestParallelPhasedSnapLoad(t *testing.T) { go func(i int) { defer vmGroup.Done() vmID := fmt.Sprintf("%d", i+vmIDBase) - message, err := orch.ResumeVM(ctx, vmID) + message, _, err := orch.ResumeVM(ctx, vmID) require.NoError(t, err, "Failed to resume VM, "+vmID+", "+message) }(i) } diff --git a/ctriface/orch_options.go b/ctriface/orch_options.go index 6dfc06f74..cdd0a0486 100644 --- a/ctriface/orch_options.go +++ b/ctriface/orch_options.go @@ -48,3 +48,11 @@ func WithUPF(isUPFEnabled bool) OrchestratorOption { o.isUPFEnabled = isUPFEnabled } } + +// WithSnapshotsDir Sets the directory where +// snapshots should be stored +func WithSnapshotsDir(snapshotsDir string) OrchestratorOption { + return func(o *Orchestrator) { + o.snapshotsDir = snapshotsDir + } +} diff --git a/fccd-orchestrator.go b/fccd-orchestrator.go index 839fcade9..34f1c38f2 100644 --- a/fccd-orchestrator.go +++ b/fccd-orchestrator.go @@ -192,5 +192,6 @@ func (s *fwdServer) FwdHello(ctx context.Context, in *hpb.FwdHelloReq) (*hpb.Fwd logger := log.WithFields(log.Fields{"fID": fID, "image": imageName, "payload": payload}) logger.Debug("Received FwdHelloVM") - return funcPool.Serve(ctx, fID, imageName, payload) + resp, _, err := funcPool.Serve(ctx, fID, imageName, payload) + return resp, err } diff --git a/fccd-orchestrator_test.go b/fccd-orchestrator_test.go index f6c8109fe..1e44f5b29 100644 --- a/fccd-orchestrator_test.go +++ b/fccd-orchestrator_test.go @@ -89,7 +89,7 @@ func TestSendToFunctionSerial(t *testing.T) { funcPool = NewFuncPool(!isSaveMemoryConst, servedTh, pinnedFuncNum, isTestModeConst) for i := 0; i < 2; i++ { - resp, err := funcPool.Serve(context.Background(), fID, imageName, "world") + resp, _, err := funcPool.Serve(context.Background(), fID, imageName, "world") require.NoError(t, err, "Function returned error") if i == 0 { require.Equal(t, resp.IsColdStart, true) @@ -117,7 +117,7 @@ func TestSendToFunctionParallel(t *testing.T) { go func(i int) { defer vmGroup.Done() - resp, err := funcPool.Serve(context.Background(), fID, imageName, "world") + resp, _, err := funcPool.Serve(context.Background(), fID, imageName, "world") require.NoError(t, err, "Function returned error") require.Equal(t, resp.Payload, "Hello, world!") }(i) @@ -140,7 +140,7 @@ func TestStartSendStopTwice(t *testing.T) { for i := 0; i < 2; i++ { for k := 0; k < 2; k++ { - resp, err := funcPool.Serve(context.Background(), fID, imageName, "world") + resp, _, err := funcPool.Serve(context.Background(), fID, imageName, "world") require.NoError(t, err, "Function returned error") require.Equal(t, resp.Payload, "Hello, world!") } @@ -164,7 +164,7 @@ func TestStatsNotNumericFunction(t *testing.T) { ) funcPool = NewFuncPool(isSaveMemoryConst, servedTh, pinnedFuncNum, isTestModeConst) - resp, err := funcPool.Serve(context.Background(), fID, imageName, "world") + resp, _, err := funcPool.Serve(context.Background(), fID, imageName, "world") require.NoError(t, err, "Function returned error") require.Equal(t, resp.Payload, "Hello, world!") @@ -186,7 +186,7 @@ func TestStatsNotColdFunction(t *testing.T) { ) funcPool = NewFuncPool(isSaveMemoryConst, servedTh, pinnedFuncNum, isTestModeConst) - resp, err := funcPool.Serve(context.Background(), fID, imageName, "world") + resp, _, err := funcPool.Serve(context.Background(), fID, imageName, "world") require.NoError(t, err, "Function returned error") require.Equal(t, resp.Payload, "Hello, world!") @@ -202,14 +202,14 @@ func TestStatsNotColdFunction(t *testing.T) { func TestSaveMemorySerial(t *testing.T) { fID := "5" imageName := "ustiugov/helloworld:runner_workload" - var ( - servedTh uint64 = 40 - pinnedFuncNum int = 2 - ) + var ( + servedTh uint64 = 40 + pinnedFuncNum int = 2 + ) funcPool = NewFuncPool(isSaveMemoryConst, servedTh, pinnedFuncNum, isTestModeConst) for i := 0; i < 100; i++ { - resp, err := funcPool.Serve(context.Background(), fID, imageName, "world") + resp, _, err := funcPool.Serve(context.Background(), fID, imageName, "world") require.NoError(t, err, "Function returned error") require.Equal(t, resp.Payload, "Hello, world!") } @@ -237,7 +237,7 @@ func TestSaveMemoryParallel(t *testing.T) { go func(i int) { defer vmGroup.Done() - resp, err := funcPool.Serve(context.Background(), fID, imageName, "world") + resp, _, err := funcPool.Serve(context.Background(), fID, imageName, "world") require.NoError(t, err, "Function returned error") require.Equal(t, resp.Payload, "Hello, world!") }(i) @@ -264,7 +264,7 @@ func TestDirectStartStopVM(t *testing.T) { message, err := funcPool.AddInstance(fID, imageName) require.NoError(t, err, "This error should never happen (addInstance())"+message) - resp, err := funcPool.Serve(context.Background(), fID, imageName, "world") + resp, _, err := funcPool.Serve(context.Background(), fID, imageName, "world") require.NoError(t, err, "Function returned error") require.Equal(t, resp.Payload, "Hello, world!") @@ -300,7 +300,7 @@ func TestAllFunctions(t *testing.T) { go func(fID int, imageName, request, response string) { defer vmGroup.Done() - resp, err := funcPool.Serve(context.Background(), strconv.Itoa(8+fID), imageName, request) + resp, _, err := funcPool.Serve(context.Background(), strconv.Itoa(8+fID), imageName, request) require.NoError(t, err, "Function returned error") require.Equal(t, resp.Payload, "Hello, "+response+"!") diff --git a/functions.go b/functions.go index f5ade070d..d31170900 100644 --- a/functions.go +++ b/functions.go @@ -26,7 +26,6 @@ import ( "context" "fmt" "math/rand" - "path/filepath" "strconv" "sync" "sync/atomic" @@ -38,6 +37,7 @@ import ( log "github.com/sirupsen/logrus" hpb "github.com/ustiugov/fccd-orchestrator/helloworld" + "github.com/ustiugov/fccd-orchestrator/metrics" ) var isTestMode bool // set with a call to NewFuncPool @@ -109,7 +109,7 @@ func (p *FuncPool) getFunction(fID, imageName string) *Function { } // Serve Service RPC request by triggering the corresponding function. -func (p *FuncPool) Serve(ctx context.Context, fID, imageName, payload string) (*hpb.FwdHelloResp, error) { +func (p *FuncPool) Serve(ctx context.Context, fID, imageName, payload string) (*hpb.FwdHelloResp, *metrics.Metric, error) { f := p.getFunction(fID, imageName) return f.Serve(ctx, fID, imageName, payload) @@ -204,13 +204,16 @@ func NewFunction(fID, imageName string, Stats *Stats, servedTh uint64, isToPin b // b. The last goroutine is determined by the atomic counter: the goroutine wih syncID==0 shuts down // the instance. // c. Instance shutdown is performed asynchronously because all instances have unique IDs. -func (f *Function) Serve(ctx context.Context, fID, imageName, reqPayload string) (*hpb.FwdHelloResp, error) { - syncID := int64(-1) // default is no synchronization +func (f *Function) Serve(ctx context.Context, fID, imageName, reqPayload string) (*hpb.FwdHelloResp, *metrics.Metric, error) { + var ( + serveMetric *metrics.Metric = metrics.NewMetric() + tStart time.Time + syncID int64 = -1 // default is no synchronization + isColdStart bool = false + ) logger := log.WithFields(log.Fields{"fID": f.fID}) - isColdStart := false - if !f.isPinnedInMem { if err := f.sem.Acquire(context.Background(), 1); err != nil { logger.Panic("Failed to acquire semaphore for serving") @@ -225,7 +228,9 @@ func (f *Function) Serve(ctx context.Context, fID, imageName, reqPayload string) func() { isColdStart = true logger.Debug("Function is inactive, starting the instance...") + tStart = time.Now() f.AddInstance() + serveMetric.MetricMap[metrics.AddInstance] = metrics.ToUS(time.Since(tStart)) }) f.RLock() @@ -234,22 +239,26 @@ func (f *Function) Serve(ctx context.Context, fID, imageName, reqPayload string) // Eventually, it needs to be RPC-dependent and probably client-defined ctxFwd, cancel := context.WithDeadline(context.Background(), time.Now().Add(20*time.Second)) defer cancel() + + tStart = time.Now() resp, err := f.fwdRPC(ctxFwd, reqPayload) + serveMetric.MetricMap[metrics.FuncInvocation] = metrics.ToUS(time.Since(tStart)) + if err != nil && ctxFwd.Err() == context.Canceled { // context deadline exceeded f.RUnlock() - return &hpb.FwdHelloResp{IsColdStart: isColdStart, Payload: ""}, err + return &hpb.FwdHelloResp{IsColdStart: isColdStart, Payload: ""}, serveMetric, err } else if err != nil { if e, ok := status.FromError(err); ok { switch e.Code() { case codes.DeadlineExceeded: // deadline exceeded f.RUnlock() - return &hpb.FwdHelloResp{IsColdStart: isColdStart, Payload: ""}, err + return &hpb.FwdHelloResp{IsColdStart: isColdStart, Payload: ""}, serveMetric, err default: logger.Warn("Function returned error: ", err) f.RUnlock() - return &hpb.FwdHelloResp{IsColdStart: isColdStart, Payload: ""}, err + return &hpb.FwdHelloResp{IsColdStart: isColdStart, Payload: ""}, serveMetric, err } } else { logger.Panic("Not able to parse error returned ", err) @@ -259,15 +268,17 @@ func (f *Function) Serve(ctx context.Context, fID, imageName, reqPayload string) if !f.isPinnedInMem && syncID == 0 { logger.Debugf("Function has to shut down its instance, served %d requests", f.GetStatServed()) + tStart = time.Now() if _, err := f.RemoveInstance(false); err != nil { logger.Panic("Failed to remove instance after servedTh expired", err) } + serveMetric.MetricMap[metrics.RetireOld] = metrics.ToUS(time.Since(tStart)) f.ZeroServedStat() f.servedSyncCounter = int64(f.servedTh) // reset counter f.sem.Release(int64(f.servedTh)) } - return &hpb.FwdHelloResp{IsColdStart: isColdStart, Payload: resp.Message}, err + return &hpb.FwdHelloResp{IsColdStart: isColdStart, Payload: resp.Message}, serveMetric, err } // FwdRPC Forward the RPC to an instance, then forwards the response back. @@ -376,7 +387,7 @@ func (f *Function) CreateInstanceSnapshot() { if err != nil { log.Panic(message, err) } - message, err = orch.CreateSnapshot(ctx, f.vmID, f.getSnapshotFilePath(), f.getMemFilePath()) + message, err = orch.CreateSnapshot(ctx, f.vmID) if err != nil { log.Panic(message, err) } @@ -415,12 +426,12 @@ func (f *Function) LoadInstance() { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() - message, err := orch.LoadSnapshot(ctx, f.vmID, f.getSnapshotFilePath(), f.getMemFilePath()) + message, _, err := orch.LoadSnapshot(ctx, f.vmID) if err != nil { log.Panic(message, err) } - message, err = orch.ResumeVM(ctx, f.vmID) + message, _, err = orch.ResumeVM(ctx, f.vmID) if err != nil { log.Panic(message, err) } @@ -440,13 +451,3 @@ func (f *Function) ZeroServedStat() { func (f *Function) getVMID() string { return fmt.Sprintf("%s_%d", f.fID, f.lastInstanceID) } - -// getSnapshotFilePath Creates the snapshot file path for the function -func (f *Function) getSnapshotFilePath() string { - return fmt.Sprintf(filepath.Join("/dev", "snap_file_%s"), f.vmID) -} - -// getMemFilePath Creates the memory file path for the function -func (f *Function) getMemFilePath() string { - return fmt.Sprintf(filepath.Join("/dev", "mem_file_%s"), f.vmID) -} diff --git a/go.mod b/go.mod index 9cea5b1e7..b97bf789d 100644 --- a/go.mod +++ b/go.mod @@ -13,9 +13,10 @@ require ( github.com/containerd/containerd v1.3.6 github.com/golang/protobuf v1.3.3 github.com/sirupsen/logrus v1.6.0 - github.com/stretchr/testify v1.5.1 - github.com/ustiugov/fccd-orchestrator/ctriface v0.0.0-20200722073430-90c1342030ac + github.com/stretchr/testify v1.6.1 + github.com/ustiugov/fccd-orchestrator/ctriface v0.0.0-20200727123940-f2d08f161c49 github.com/ustiugov/fccd-orchestrator/helloworld v0.0.0-20200717125634-528c6e9f9cc9 + github.com/ustiugov/fccd-orchestrator/metrics v0.0.0-20200727123940-f2d08f161c49 github.com/ustiugov/fccd-orchestrator/misc v0.0.0-20200717125634-528c6e9f9cc9 // indirect github.com/ustiugov/fccd-orchestrator/proto v0.0.0-20200717125634-528c6e9f9cc9 github.com/ustiugov/fccd-orchestrator/taps v0.0.0-20200717125634-528c6e9f9cc9 // indirect diff --git a/go.sum b/go.sum index 82063efb1..fd090f720 100644 --- a/go.sum +++ b/go.sum @@ -16,6 +16,7 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= @@ -104,6 +105,7 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/firecracker-microvm/firecracker-containerd v0.0.0-20200331220105-afedbc74f5ee h1:SKPxUz0fLm3Mf/ZWBTUHuXp/2vLHf+UvtxW3W5KNt/s= github.com/firecracker-microvm/firecracker-containerd v0.0.0-20200331220105-afedbc74f5ee/go.mod h1:7B+eRdKxqB3zSSsXsz5Jp+cGXrLgdyQIhEkT1Pfj+Zk= github.com/firecracker-microvm/firecracker-go-sdk v0.21.0/go.mod h1:zyc9BrKGePpNLbQ5y2ZtdzXEfpMJeHPeFNVpyo0S1WQ= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= @@ -174,6 +176,7 @@ github.com/gogo/protobuf v1.3.0 h1:G8O7TerXerS4F6sx9OV7/nRfJdnXgHZu/S/7F2SN+UE= github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -212,6 +215,7 @@ github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6t github.com/juju/errors v0.0.0-20180806074554-22422dad46e1/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= github.com/juju/testing v0.0.0-20190613124551-e81189438503/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -296,6 +300,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 h1:b6uOv7YOFK0TYG7HtkIgExQo+2RdLuwRft63jn2HWj8= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= @@ -373,6 +379,16 @@ github.com/ustiugov/fccd-orchestrator/ctriface v0.0.0-20200722072720-7c2beba403d github.com/ustiugov/fccd-orchestrator/ctriface v0.0.0-20200722072720-7c2beba403de/go.mod h1:UcNV4o8382RvEIe3gb19tIA7wRcZ+Q0DCbmSUCl7tPM= github.com/ustiugov/fccd-orchestrator/ctriface v0.0.0-20200722073430-90c1342030ac h1:r+eb9hmonXVp1R3X6Irp9kY3kU0sA71cZFPxSLZAvZ0= github.com/ustiugov/fccd-orchestrator/ctriface v0.0.0-20200722073430-90c1342030ac/go.mod h1:UcNV4o8382RvEIe3gb19tIA7wRcZ+Q0DCbmSUCl7tPM= +github.com/ustiugov/fccd-orchestrator/ctriface v0.0.0-20200723101147-e9f1a1f5bf4c h1:feusW90Da1zUR2jwxvVfIEYK2B4HUv71+luKk8X9Z6E= +github.com/ustiugov/fccd-orchestrator/ctriface v0.0.0-20200723101147-e9f1a1f5bf4c/go.mod h1:XbU2ULnOfHWIdbLhyp4zaieQ0cIrvUzP9aDsgq3GkGQ= +github.com/ustiugov/fccd-orchestrator/ctriface v0.0.0-20200723105930-96d5e1cb279b h1:jOckr1RkPU23LYyLmwXK30TcEIn21RAAr4fLT4i+kMc= +github.com/ustiugov/fccd-orchestrator/ctriface v0.0.0-20200723105930-96d5e1cb279b/go.mod h1:XbU2ULnOfHWIdbLhyp4zaieQ0cIrvUzP9aDsgq3GkGQ= +github.com/ustiugov/fccd-orchestrator/ctriface v0.0.0-20200724131919-841ab08744c3 h1:8QgAi9bSaM+6MvfedOrcGMHpxeQmkSzEL61wcKJoMPE= +github.com/ustiugov/fccd-orchestrator/ctriface v0.0.0-20200724131919-841ab08744c3/go.mod h1:Lu5pFaIlBMutG2QIvxt0aJU2n6vV2HDONmFwMmt9SU0= +github.com/ustiugov/fccd-orchestrator/ctriface v0.0.0-20200727115924-ea27568b4c04 h1:8Jr2LdVcgqAjTo/kXy1cVi7ffFeSzC2PAjidN+M1wYM= +github.com/ustiugov/fccd-orchestrator/ctriface v0.0.0-20200727115924-ea27568b4c04/go.mod h1:Ugov5iZ6u7XYLBB+Y4S1Tdmma7U4ykupjHksg9iJ6Vc= +github.com/ustiugov/fccd-orchestrator/ctriface v0.0.0-20200727123940-f2d08f161c49 h1:Umb7+S76V7y3uyYEPx1sHx1DM0lcsMhMe0757bPaseg= +github.com/ustiugov/fccd-orchestrator/ctriface v0.0.0-20200727123940-f2d08f161c49/go.mod h1:/LM7fpMbZXS1CV6N4SPeiXXTFbxiEj6ZjZIIjLUamtk= github.com/ustiugov/fccd-orchestrator/helloworld v0.0.0-20200710144657-9fbec6857e48/go.mod h1:5dhCs/XynpQoQcrhd/YgUBjGahhNpTknQUcC1kHRCaA= github.com/ustiugov/fccd-orchestrator/helloworld v0.0.0-20200710145415-bb09d1a68889/go.mod h1:5dhCs/XynpQoQcrhd/YgUBjGahhNpTknQUcC1kHRCaA= github.com/ustiugov/fccd-orchestrator/helloworld v0.0.0-20200710150633-096cac68bd72 h1:r67pqykSYWZHFYfIKuT44PTXFJWu5lJcatkV16d3vKU= @@ -386,6 +402,18 @@ github.com/ustiugov/fccd-orchestrator/helloworld v0.0.0-20200714162243-d6dc0c083 github.com/ustiugov/fccd-orchestrator/helloworld v0.0.0-20200714162243-d6dc0c083e9e/go.mod h1:5dhCs/XynpQoQcrhd/YgUBjGahhNpTknQUcC1kHRCaA= github.com/ustiugov/fccd-orchestrator/helloworld v0.0.0-20200717125634-528c6e9f9cc9 h1:1EQvsISd3xbC1krrXCCuIsingP9PF4M6pfCTXPnhYb4= github.com/ustiugov/fccd-orchestrator/helloworld v0.0.0-20200717125634-528c6e9f9cc9/go.mod h1:5dhCs/XynpQoQcrhd/YgUBjGahhNpTknQUcC1kHRCaA= +github.com/ustiugov/fccd-orchestrator/metrics v0.0.0-20200723085111-a9d697412510 h1:2F3Bp/P7C2UMKU+vakjEW/++E2yCntucHKNiRxcxQOE= +github.com/ustiugov/fccd-orchestrator/metrics v0.0.0-20200723085111-a9d697412510/go.mod h1:NOrYF6usVkdr6gpXAP7pk7jp1MlAXlrZQuUEOVhRuV8= +github.com/ustiugov/fccd-orchestrator/metrics v0.0.0-20200723105930-96d5e1cb279b h1:0ukOv2nUQqj14TLHf5MkVyvbyiV6AxwUSI7PQQnVdKQ= +github.com/ustiugov/fccd-orchestrator/metrics v0.0.0-20200723105930-96d5e1cb279b/go.mod h1:NOrYF6usVkdr6gpXAP7pk7jp1MlAXlrZQuUEOVhRuV8= +github.com/ustiugov/fccd-orchestrator/metrics v0.0.0-20200724084859-4128c20228f6/go.mod h1:NOrYF6usVkdr6gpXAP7pk7jp1MlAXlrZQuUEOVhRuV8= +github.com/ustiugov/fccd-orchestrator/metrics v0.0.0-20200724131919-841ab08744c3 h1:j+OAhC9VvLfo/JwPbP7HGRQkhw5F/3Yf/Z2Y6SqIQCI= +github.com/ustiugov/fccd-orchestrator/metrics v0.0.0-20200724131919-841ab08744c3/go.mod h1:NOrYF6usVkdr6gpXAP7pk7jp1MlAXlrZQuUEOVhRuV8= +github.com/ustiugov/fccd-orchestrator/metrics v0.0.0-20200727100916-b838e9b56a67/go.mod h1:NOrYF6usVkdr6gpXAP7pk7jp1MlAXlrZQuUEOVhRuV8= +github.com/ustiugov/fccd-orchestrator/metrics v0.0.0-20200727115924-ea27568b4c04 h1:H5PUKmgr+tRmq48OPnfVJfFoSwD7YC1tYIhaT2MjRDQ= +github.com/ustiugov/fccd-orchestrator/metrics v0.0.0-20200727115924-ea27568b4c04/go.mod h1:kqi59gkrrgkSH4W9PH85SL00/kLZysF4NlmmpZFLs58= +github.com/ustiugov/fccd-orchestrator/metrics v0.0.0-20200727123940-f2d08f161c49 h1:/V0J6ZK8XR/mBcBm27aBvEIpDbhwsHglyiI7OHk3RBg= +github.com/ustiugov/fccd-orchestrator/metrics v0.0.0-20200727123940-f2d08f161c49/go.mod h1:kqi59gkrrgkSH4W9PH85SL00/kLZysF4NlmmpZFLs58= github.com/ustiugov/fccd-orchestrator/misc v0.0.0-20200710145415-bb09d1a68889/go.mod h1:Fz+jqGMI/infxs9IhgBZbmW/dtZ7odhhD8cg6W+gAaA= github.com/ustiugov/fccd-orchestrator/misc v0.0.0-20200710150633-096cac68bd72 h1:yDwPSYKCtJaqI2NqVD0ugI7cnCg+ynnEOiVoLXHcWOg= github.com/ustiugov/fccd-orchestrator/misc v0.0.0-20200710150633-096cac68bd72/go.mod h1:Fz+jqGMI/infxs9IhgBZbmW/dtZ7odhhD8cg6W+gAaA= @@ -444,7 +472,11 @@ golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -512,11 +544,13 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -531,9 +565,16 @@ golang.org/x/tools v0.0.0-20200717024301-6ddee64345a6 h1:nULzSsKgihxFGLnQFv2T7lE golang.org/x/tools v0.0.0-20200717024301-6ddee64345a6/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200721032237-77f530d86f9a/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200721223218-6123e77877b2/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200723000907-a7c6fd066f6d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200725200936-102e7d357031/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.7.0 h1:Hdks0L0hgznZLG9nzXb8vZ0rRvqNvAcgAp84y7Mwkgw= +gonum.org/v1/gonum v0.7.0/go.mod h1:L02bwd0sqlsvRv41G7wGWFCsVNZFv/k1xzGIxeANHGM= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -568,8 +609,11 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/manual_cleanup_test.go b/manual_cleanup_test.go index adae2ec55..688041186 100644 --- a/manual_cleanup_test.go +++ b/manual_cleanup_test.go @@ -41,7 +41,7 @@ func TestParallelServe(t *testing.T) { funcPool = NewFuncPool(isSaveMemoryConst, servedTh, pinnedFuncNum, isTestModeConst) // Pull image to work around parallel pulling limitation - resp, err := funcPool.Serve(context.Background(), "plr_fnc", imageName, "world") + resp, _, err := funcPool.Serve(context.Background(), "plr_fnc", imageName, "world") require.NoError(t, err, "Function returned error") require.Equal(t, resp.Payload, "Hello, world!") // ----------------------------------------------------- @@ -55,11 +55,11 @@ func TestParallelServe(t *testing.T) { defer vmGroup.Done() fID := strconv.Itoa(15 + i) - resp, err := funcPool.Serve(context.Background(), fID, imageName, "world") + resp, _, err := funcPool.Serve(context.Background(), fID, imageName, "world") require.NoError(t, err, "Function returned error on 1st run") require.Equal(t, resp.Payload, "Hello, world!") - resp, err = funcPool.Serve(context.Background(), fID, imageName, "world") + resp, _, err = funcPool.Serve(context.Background(), fID, imageName, "world") require.NoError(t, err, "Function returned error on 2nd run") require.Equal(t, resp.Payload, "Hello, world!") }(i) @@ -76,16 +76,16 @@ func TestServeThree(t *testing.T) { ) funcPool = NewFuncPool(isSaveMemoryConst, servedTh, pinnedFuncNum, isTestModeConst) - resp, err := funcPool.Serve(context.Background(), fID, imageName, "world") + resp, _, err := funcPool.Serve(context.Background(), fID, imageName, "world") require.NoError(t, err, "Function returned error on 1st run") require.Equal(t, resp.IsColdStart, true) require.Equal(t, resp.Payload, "Hello, world!") - resp, err = funcPool.Serve(context.Background(), fID, imageName, "world") + resp, _, err = funcPool.Serve(context.Background(), fID, imageName, "world") require.NoError(t, err, "Function returned error on 2nd run") require.Equal(t, resp.Payload, "Hello, world!") - resp, err = funcPool.Serve(context.Background(), fID, imageName, "world") + resp, _, err = funcPool.Serve(context.Background(), fID, imageName, "world") require.NoError(t, err, "Function returned error on 3rd run") require.Equal(t, resp.Payload, "Hello, world!") } diff --git a/metrics/go.mod b/metrics/go.mod new file mode 100644 index 000000000..55a1bd6a0 --- /dev/null +++ b/metrics/go.mod @@ -0,0 +1,10 @@ +module github.com/ustiugov/fccd-orchestrator/metrics + +go 1.14 + +require ( + github.com/stretchr/testify v1.6.1 + golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect + golang.org/x/tools v0.0.0-20200725200936-102e7d357031 // indirect + gonum.org/v1/gonum v0.7.0 +) diff --git a/metrics/go.sum b/metrics/go.sum new file mode 100644 index 000000000..926cfade4 --- /dev/null +++ b/metrics/go.sum @@ -0,0 +1,55 @@ +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7 h1:EBZoQjiKKPaLbPrbpssUfuHtwM6KV/vb4U85g/cigFY= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200723000907-a7c6fd066f6d h1:7k9BKfwmdbykG6l5ztniTrH0TP25yel8O7l26/yovMU= +golang.org/x/tools v0.0.0-20200723000907-a7c6fd066f6d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200725200936-102e7d357031 h1:VtIxiVHWPhnny2ZTi4f9/2diZKqyLaq3FUTuud5+khA= +golang.org/x/tools v0.0.0-20200725200936-102e7d357031/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.7.0 h1:Hdks0L0hgznZLG9nzXb8vZ0rRvqNvAcgAp84y7Mwkgw= +gonum.org/v1/gonum v0.7.0/go.mod h1:L02bwd0sqlsvRv41G7wGWFCsVNZFv/k1xzGIxeANHGM= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/metrics/metrics.go b/metrics/metrics.go new file mode 100644 index 000000000..bf980da5b --- /dev/null +++ b/metrics/metrics.go @@ -0,0 +1,160 @@ +// MIT License +// +// Copyright (c) 2020 Plamen Petrov +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package metrics + +import ( + "bufio" + "fmt" + "os" + "sort" + "time" + + "gonum.org/v1/gonum/stat" +) + +const ( + // FcResume Time it takes to resume a VM from containerd + FcResume = "FcResume" + // ReconnectFuncClient Time it takes to reconnect function client + ReconnectFuncClient = "ReconnectFuncClient" + + // Full Used when there is no breakdown + Full = "Full" + + // AddInstance Time to add instance - load snap or start vm + AddInstance = "AddInstance" + // FuncInvocation Time to get response from function + FuncInvocation = "FuncInvocation" + // RetireOld Time to offload/stop instance if threshold exceeded + RetireOld = "RetireOld" + + // GetImage Time to pull docker image + GetImage = "GetImage" + // FcCreateVM Time to create VM + FcCreateVM = "FcCreateVM" + // NewContainer Time to create new container + NewContainer = "NewContainer" + // NewTask Time to create new task + NewTask = "NewTask" + // TaskWait Time to wait for task to be ready + TaskWait = "TaskWait" + // TaskStart Time to start task + TaskStart = "TaskStart" +) + +// Metric A general metric +type Metric struct { + MetricMap map[string]float64 +} + +// NewMetric Create a new metric +func NewMetric() *Metric { + m := new(Metric) + m.MetricMap = make(map[string]float64) + + return m +} + +// Total Calculates the total time per stat +func (m *Metric) Total() float64 { + var sum float64 + for _, v := range m.MetricMap { + sum += v + } + + return sum +} + +// PrintTotal Prints the total time +func (m *Metric) PrintTotal() { + fmt.Printf("Total: %.1f us\n", m.Total()) +} + +// PrintAll Prints a breakdown of the time +func (m *Metric) PrintAll() { + for k, v := range m.MetricMap { + fmt.Printf("%s:\t%.1f\n", k, v) + } + fmt.Printf("Total\t%.1f\n", m.Total()) +} + +// PrintMeanStd prints the mean and standard +//deviation of each component of Metric +func PrintMeanStd(resultsPath string, metricsList ...*Metric) { + var ( + mean, std float64 + f *os.File + err error + agg map[string][]float64 = make(map[string][]float64) + totals []float64 = make([]float64, 0, len(metricsList)) + keys []string = make([]string, 0, 1) + ) + + if len(metricsList) == 0 { + return + } + + if resultsPath == "" { + f = os.Stdout + } else { + f, err = os.Create(resultsPath) + if err != nil { + panic(err) + } + defer f.Close() + } + + for k := range metricsList[0].MetricMap { + keys = append(keys, k) + agg[k] = make([]float64, 0, len(metricsList)) + } + sort.Strings(keys) + + for _, m := range metricsList { + totals = append(totals, m.Total()) + + for k, v := range m.MetricMap { + agg[k] = append(agg[k], v) + } + } + + w := bufio.NewWriter(f) + + fmt.Fprintf(w, "Stats \tMean(us) \tStdDev(us)\n") + + for _, k := range keys { + v := agg[k] + mean, std = stat.MeanStdDev(v, nil) + fmt.Fprintf(w, "%s \t%12.1f \t%12.1f\n", k, mean, std) + w.Flush() + } + + mean, std = stat.MeanStdDev(totals, nil) + fmt.Fprintf(w, "Total \t%12.1f \t%12.1f\n", mean, std) + w.Flush() +} + +// ToUS Converts Duration to microseconds +func ToUS(dur time.Duration) float64 { + return float64(dur.Microseconds()) +} diff --git a/metrics/metrics_test.go b/metrics/metrics_test.go new file mode 100644 index 000000000..6cef9acf2 --- /dev/null +++ b/metrics/metrics_test.go @@ -0,0 +1,42 @@ +// MIT License +// +// Copyright (c) 2020 Plamen Petrov +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package metrics + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMetrics(t *testing.T) { + s1 := NewMetric() + s1.MetricMap[GetImage] = 10.0 + s1.MetricMap[TaskStart] = 15.0 + require.Equal(t, float64(25.0), s1.Total(), "Total is incorrect") + + s2 := NewMetric() + s2.MetricMap[GetImage] = 40.0 + s2.MetricMap[TaskStart] = 25.0 + + PrintMeanStd("", s1, s2) +}