diff --git a/memory/manager/go.mod b/memory/manager/go.mod index 11e73df38..a775d21f5 100644 --- a/memory/manager/go.mod +++ b/memory/manager/go.mod @@ -18,8 +18,9 @@ require ( github.com/opencontainers/runc v0.1.1 // indirect github.com/opencontainers/runtime-spec v1.0.2 // indirect github.com/sirupsen/logrus v1.6.0 - github.com/stretchr/testify v1.4.0 + github.com/stretchr/testify v1.6.1 github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 // indirect + github.com/ustiugov/fccd-orchestrator/metrics v0.0.0-20200812154705-d5dc53e49664 golang.org/x/net v0.0.0-20200707034311-ab3426394381 // indirect golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 // indirect golang.org/x/sys v0.0.0-20200805065543-0cf7623e9dbd diff --git a/memory/manager/go.sum b/memory/manager/go.sum index 01c51d68e..fb32ccbcb 100644 --- a/memory/manager/go.sum +++ b/memory/manager/go.sum @@ -117,13 +117,19 @@ github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 h1:b6uOv7YOFK0TYG7HtkIgExQo+2RdLuwRft63jn2HWj8= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/ustiugov/fccd-orchestrator v0.0.0-20200812154705-d5dc53e49664 h1:EJ2XH9BfK/+V96OY879w49CmbaIvwLnxlRuc+M3vsAs= +github.com/ustiugov/fccd-orchestrator/metrics v0.0.0-20200812154705-d5dc53e49664 h1:2zNuh9/TfIuTHO1cL1Dje50FfK6ncw2/gllBI+DcHMY= +github.com/ustiugov/fccd-orchestrator/metrics v0.0.0-20200812154705-d5dc53e49664/go.mod h1:kqi59gkrrgkSH4W9PH85SL00/kLZysF4NlmmpZFLs58= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -133,6 +139,9 @@ golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86h golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -140,7 +149,9 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -178,6 +189,11 @@ golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200725200936-102e7d357031/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.7.0 h1:Hdks0L0hgznZLG9nzXb8vZ0rRvqNvAcgAp84y7Mwkgw= @@ -210,6 +226,7 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/memory/manager/manager.go b/memory/manager/manager.go index 57f4c4e0f..deea0005b 100644 --- a/memory/manager/manager.go +++ b/memory/manager/manager.go @@ -9,6 +9,7 @@ import ( "sync" "syscall" + "github.com/ustiugov/fccd-orchestrator/metrics" "gonum.org/v1/gonum/stat" log "github.com/sirupsen/logrus" @@ -128,9 +129,13 @@ func (m *MemoryManager) Activate(vmID string, userFaultFDFile *os.File) (err err state.isEverActivated = true state.firstPageFaultOnce = new(sync.Once) state.servedNum = 0 - state.uniqueNum = 0 state.quitCh = make(chan int) + if state.metricsModeOn { + state.uniqueNum = 0 + state.currentMetric = metrics.NewMetric() + } + fdInt = int(state.userFaultFD.Fd()) event.Events = syscall.EPOLLIN @@ -197,6 +202,7 @@ func (m *MemoryManager) Deactivate(vmID string) error { state.reusedPFServed, float64(state.servedNum-state.uniqueNum), ) + state.latencyMetrics = append(state.latencyMetrics, state.currentMetric) } state.userFaultFD.Close() diff --git a/memory/manager/snapshot_state.go b/memory/manager/snapshot_state.go index d331cd4ef..720ec1542 100644 --- a/memory/manager/snapshot_state.go +++ b/memory/manager/snapshot_state.go @@ -23,6 +23,8 @@ import ( log "github.com/sirupsen/logrus" "golang.org/x/sys/unix" + "github.com/ustiugov/fccd-orchestrator/metrics" + "unsafe" ) @@ -65,9 +67,6 @@ type SnapshotState struct { isWSCopy bool - servedNum int - uniqueNum int - guestMem []byte workingSet []byte @@ -75,6 +74,11 @@ type SnapshotState struct { totalPFServed []float64 uniquePFServed []float64 reusedPFServed []float64 + latencyMetrics []*metrics.Metric + + servedNum int + uniqueNum int + currentMetric *metrics.Metric } // NewSnapshotState Initializes a snapshot state @@ -87,6 +91,7 @@ func NewSnapshotState(cfg SnapshotStateCfg) *SnapshotState { s.totalPFServed = make([]float64, 0) s.uniquePFServed = make([]float64, 0) s.reusedPFServed = make([]float64, 0) + s.latencyMetrics = make([]*metrics.Metric, 0) } return s @@ -222,6 +227,7 @@ func (s *SnapshotState) pollUserPageFaults(readyCh chan int) { var ( events [1]syscall.EpollEvent + tStart time.Time ) logger.Debug("Starting polling loop") @@ -232,7 +238,14 @@ func (s *SnapshotState) pollUserPageFaults(readyCh chan int) { // if s.isReplayWorkingSet { if s.isRecordReady && !s.IsLazyMode { - s.fetchState() + if s.metricsModeOn { + tStart = time.Now() + s.fetchState() + s.currentMetric.MetricMap["FetchState"] = metrics.ToUS(time.Since(tStart)) + } else { + s.fetchState() + } + } // } @@ -287,7 +300,11 @@ func (s *SnapshotState) pollUserPageFaults(readyCh chan int) { } func (s *SnapshotState) servePageFault(fd int, address uint64) error { - var workingSetInstalled bool + var ( + tStart time.Time + serveUnique string = "ServeUnique" + workingSetInstalled bool + ) s.firstPageFaultOnce.Do( func() { @@ -295,7 +312,15 @@ func (s *SnapshotState) servePageFault(fd int, address uint64) error { // if s.isReplayWorkingSet if s.isRecordReady && !s.IsLazyMode { - s.installWorkingSetPages(fd) + if s.metricsModeOn { + tStart = time.Now() + s.installWorkingSetPages(fd) + s.currentMetric.MetricMap["InstallWS"] = metrics.ToUS(time.Since(tStart)) + s.currentMetric.MetricMap[serveUnique] = float64(0) + } else { + s.installWorkingSetPages(fd) + } + workingSetInstalled = true } // } @@ -330,9 +355,17 @@ func (s *SnapshotState) servePageFault(fd int, address uint64) error { } s.servedNum++ + + tStart = time.Now() + } + + err := installRegion(fd, src, dst, mode, 1) + + if s.metricsModeOn { + s.currentMetric.MetricMap[serveUnique] += metrics.ToUS(time.Since(tStart)) } - return installRegion(fd, src, dst, mode, 1) + return err } func (s *SnapshotState) installWorkingSetPages(fd int) {