From 2e91fea4424e94183cfa2f5445fda0185014e246 Mon Sep 17 00:00:00 2001 From: Anirudh Aithal Date: Wed, 25 Jan 2017 08:06:40 -0800 Subject: [PATCH 01/27] Fixed some race conditions in task engine tests (#673) * Fix event state race condition in task engine unit tests. I think this originated from copying test code from a unit test that already had this bug because of the race condition. The deleted line of code used to result in race condition and it would trigger the task engine to do additional work after the test would complete leading to test failures. Should fix #669 * Vendor gomock with support for MinTimes * Add documentation for using MinTimes in TestTaskTransitionWhenStopContainerReturnsUnretriableError test --- agent/Godeps/Godeps.json | 4 +- agent/api/types.go | 2 +- agent/engine/docker_task_engine_test.go | 79 +- agent/vendor/github.com/docker/docker/AUTHORS | 1652 +++++++++++++++++ .../docker/docker/project/CONTRIBUTORS.md | 1 + agent/vendor/github.com/golang/mock/AUTHORS | 12 + .../github.com/golang/mock/CONTRIBUTORS | 37 + .../github.com/golang/mock/gomock/call.go | 21 + agent/vendor/golang.org/x/net/AUTHORS | 3 + agent/vendor/golang.org/x/net/CONTRIBUTORS | 3 + agent/vendor/golang.org/x/sys/AUTHORS | 3 + agent/vendor/golang.org/x/sys/CONTRIBUTORS | 3 + agent/vendor/golang.org/x/tools/AUTHORS | 3 + agent/vendor/golang.org/x/tools/CONTRIBUTORS | 3 + 14 files changed, 1781 insertions(+), 45 deletions(-) create mode 100644 agent/vendor/github.com/docker/docker/AUTHORS create mode 120000 agent/vendor/github.com/docker/docker/project/CONTRIBUTORS.md create mode 100644 agent/vendor/github.com/golang/mock/AUTHORS create mode 100644 agent/vendor/github.com/golang/mock/CONTRIBUTORS create mode 100644 agent/vendor/golang.org/x/net/AUTHORS create mode 100644 agent/vendor/golang.org/x/net/CONTRIBUTORS create mode 100644 agent/vendor/golang.org/x/sys/AUTHORS create mode 100644 agent/vendor/golang.org/x/sys/CONTRIBUTORS create mode 100644 agent/vendor/golang.org/x/tools/AUTHORS create mode 100644 agent/vendor/golang.org/x/tools/CONTRIBUTORS diff --git a/agent/Godeps/Godeps.json b/agent/Godeps/Godeps.json index e0fbf59c123..2b92d6239d8 100644 --- a/agent/Godeps/Godeps.json +++ b/agent/Godeps/Godeps.json @@ -1,6 +1,6 @@ { "ImportPath": "github.com/aws/amazon-ecs-agent/agent", - "GoVersion": "go1.6", + "GoVersion": "go1.7", "GodepVersion": "v75", "Packages": [ "./..." @@ -257,7 +257,7 @@ }, { "ImportPath": "github.com/golang/mock/gomock", - "Rev": "15f8b22550555c0d3edf5afa97d74001bda2208b" + "Rev": "bd3c8e81be01eef76d4b503f5e687d2d1354d2d9" }, { "ImportPath": "github.com/gorilla/websocket", diff --git a/agent/api/types.go b/agent/api/types.go index d0b63864164..b1e468b1ed9 100644 --- a/agent/api/types.go +++ b/agent/api/types.go @@ -206,7 +206,7 @@ func (t *TaskStateChange) String() string { } func (t *Task) String() string { - res := fmt.Sprintf("%s:%s %s, Status: (%s->%s)", t.Family, t.Version, t.Arn, t.GetKnownStatus().String(), t.DesiredStatus.String()) + res := fmt.Sprintf("%s:%s %s, Status: (%s->%s)", t.Family, t.Version, t.Arn, t.GetKnownStatus().String(), t.GetDesiredStatus().String()) res += " Containers: [" for _, c := range t.Containers { res += fmt.Sprintf("%s (%s->%s),", c.Name, c.GetKnownStatus().String(), c.GetDesiredStatus().String()) diff --git a/agent/engine/docker_task_engine_test.go b/agent/engine/docker_task_engine_test.go index cc6523c5fad..228551e0add 100644 --- a/agent/engine/docker_task_engine_test.go +++ b/agent/engine/docker_task_engine_test.go @@ -402,13 +402,11 @@ func TestStartTimeoutThenStart(t *testing.T) { if contEvent.Status != api.ContainerStopped { t.Fatal("Expected container to timeout on start and stop") } - *contEvent.SentStatus = api.ContainerStopped taskEvent := <-taskEvents if taskEvent.Status != api.TaskStopped { t.Fatal("And then task") } - *taskEvent.SentStatus = api.TaskStopped select { case <-taskEvents: t.Fatal("Should be out of events") @@ -763,13 +761,11 @@ func TestTaskTransitionWhenStopContainerTimesout(t *testing.T) { if contEvent.Status != api.ContainerRunning { t.Errorf("Expected container to be running, got: %v", contEvent) } - *contEvent.SentStatus = api.ContainerRunning taskEvent := <-taskEvents if taskEvent.Status != api.TaskRunning { t.Errorf("Expected task to be running, got: %v", taskEvent) } - *taskEvent.SentStatus = api.TaskRunning // Set the task desired status to be stopped and StopContainer will be called updateSleepTask := *sleepTask @@ -798,12 +794,10 @@ func TestTaskTransitionWhenStopContainerTimesout(t *testing.T) { if contEvent.Status != api.ContainerStopped { t.Errorf("Expected container to timeout on start and stop, got: %v", contEvent) } - *contEvent.SentStatus = api.ContainerStopped taskEvent = <-taskEvents if taskEvent.Status != api.TaskStopped { t.Errorf("Expected task to be stopped, got: %v", taskEvent) } - *taskEvent.SentStatus = api.TaskStopped select { case <-taskEvents: @@ -819,7 +813,6 @@ func TestTaskTransitionWhenStopContainerTimesout(t *testing.T) { // stop container call returns an unretriable error from docker, specifically the // ContainerNotRunning error func TestTaskTransitionWhenStopContainerReturnsUnretriableError(t *testing.T) { - t.Skip("Skipping test with high false-positive rate.") ctrl, client, mockTime, taskEngine, _, imageManager := mocks(t, &defaultConfig) defer ctrl.Finish() @@ -835,6 +828,7 @@ func TestTaskTransitionWhenStopContainerReturnsUnretriableError(t *testing.T) { client.EXPECT().Version() client.EXPECT().ContainerEvents(gomock.Any()).Return(eventStream, nil) mockTime.EXPECT().After(gomock.Any()).AnyTimes() + eventsReported := sync.WaitGroup{} for _, container := range sleepTask.Containers { gomock.InOrder( imageManager.EXPECT().AddAllImageStates(gomock.Any()).AnyTimes(), @@ -844,20 +838,32 @@ func TestTaskTransitionWhenStopContainerReturnsUnretriableError(t *testing.T) { // Simulate successful create container client.EXPECT().CreateContainer(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Do( func(x, y, z, timeout interface{}) { - go func() { eventStream <- dockerEvent(api.ContainerCreated) }() + eventsReported.Add(1) + go func() { + eventStream <- dockerEvent(api.ContainerCreated) + eventsReported.Done() + }() }).Return(DockerContainerMetadata{DockerID: "containerId"}), // Simulate successful start container client.EXPECT().StartContainer("containerId", startContainerTimeout).Do( func(id string, timeout time.Duration) { + eventsReported.Add(1) go func() { eventStream <- dockerEvent(api.ContainerRunning) + eventsReported.Done() }() }).Return(DockerContainerMetadata{DockerID: "containerId"}), // StopContainer errors out. However, since this is a known unretriable error, // the task engine should not retry stopping the container and move on. + // If there's a delay in task engine's processing of the ContainerRunning + // event, StopContainer will be invoked again as the engine considers it + // as a stopped container coming back. MinTimes() should guarantee that + // StopContainer is invoked at least once and in protecting agasint a test + // failure when there's a delay in task engine processing the ContainerRunning + // event. client.EXPECT().StopContainer("containerId", gomock.Any()).Return(DockerContainerMetadata{ Error: CannotStopContainerError{&docker.ContainerNotRunning{}}, - }), + }).MinTimes(1), ) } @@ -869,11 +875,18 @@ func TestTaskTransitionWhenStopContainerReturnsUnretriableError(t *testing.T) { // wait for task running contEvent := <-contEvents assert.Equal(t, contEvent.Status, api.ContainerRunning, "Expected container to be running") - *contEvent.SentStatus = api.ContainerRunning taskEvent := <-taskEvents assert.Equal(t, taskEvent.Status, api.TaskRunning, "Expected task to be running") - *taskEvent.SentStatus = api.TaskRunning + + select { + case <-taskEvents: + t.Fatal("Should be out of events") + case <-contEvents: + t.Fatal("Should be out of events") + default: + } + eventsReported.Wait() // Set the task desired status to be stopped and StopContainer will be called updateSleepTask := *sleepTask @@ -884,10 +897,8 @@ func TestTaskTransitionWhenStopContainerReturnsUnretriableError(t *testing.T) { // Expect it to go to stopped contEvent = <-contEvents assert.Equal(t, contEvent.Status, api.ContainerStopped, "Expected container to be stopped") - *contEvent.SentStatus = api.ContainerStopped taskEvent = <-taskEvents assert.Equal(t, taskEvent.Status, api.TaskStopped, "Expected task to be stopped") - *taskEvent.SentStatus = api.TaskStopped select { case <-taskEvents: @@ -907,12 +918,6 @@ func TestTaskTransitionWhenStopContainerReturnsTransientErrorBeforeSucceeding(t sleepTask := testdata.LoadTask("sleep5") eventStream := make(chan DockerContainerChangeEvent) - dockerEvent := func(status api.ContainerStatus) DockerContainerChangeEvent { - meta := DockerContainerMetadata{ - DockerID: "containerId", - } - return DockerContainerChangeEvent{Status: status, DockerContainerMetadata: meta} - } client.EXPECT().Version() client.EXPECT().ContainerEvents(gomock.Any()).Return(eventStream, nil) @@ -927,32 +932,18 @@ func TestTaskTransitionWhenStopContainerReturnsTransientErrorBeforeSucceeding(t imageManager.EXPECT().RecordContainerReference(container), imageManager.EXPECT().GetImageStateFromImageName(gomock.Any()).Return(nil), // Simulate successful create container - client.EXPECT().CreateContainer(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Do( - func(x, y, z, timeout interface{}) { - go func() { eventStream <- dockerEvent(api.ContainerCreated) }() - }).Return(DockerContainerMetadata{DockerID: "containerId"}), + client.EXPECT().CreateContainer(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return( + DockerContainerMetadata{DockerID: "containerId"}), // Simulate successful start container - client.EXPECT().StartContainer("containerId", startContainerTimeout).Do( - func(id string, timeout time.Duration) { - go func() { - eventStream <- dockerEvent(api.ContainerRunning) - }() - }).Return(DockerContainerMetadata{DockerID: "containerId"}), + client.EXPECT().StartContainer("containerId", startContainerTimeout).Return( + DockerContainerMetadata{DockerID: "containerId"}), // StopContainer errors out a couple of times client.EXPECT().StopContainer("containerId", gomock.Any()).Return(containerStoppingError).Times(2), // Since task is not in steady state, progressContainers causes // another invocation of StopContainer. Return the 'succeed' response, // which should cause the task engine to stop invoking this again and // transition the task to stopped. - client.EXPECT().StopContainer("containerId", gomock.Any()).Do( - func(id string, timeout time.Duration) { - go func() { - // Emit 'ContainerStopped' event to the container event stream - // This should cause the container and the task to transition - // to 'STOPPED' - eventStream <- dockerEvent(api.ContainerStopped) - }() - }).Return(DockerContainerMetadata{}), + client.EXPECT().StopContainer("containerId", gomock.Any()).Return(DockerContainerMetadata{}), ) } @@ -964,11 +955,17 @@ func TestTaskTransitionWhenStopContainerReturnsTransientErrorBeforeSucceeding(t // wait for task running contEvent := <-contEvents assert.Equal(t, contEvent.Status, api.ContainerRunning, "Expected container to be running") - *contEvent.SentStatus = api.ContainerRunning taskEvent := <-taskEvents assert.Equal(t, taskEvent.Status, api.TaskRunning, "Expected task to be running") - *taskEvent.SentStatus = api.TaskRunning + + select { + case <-taskEvents: + t.Fatal("Should be out of events") + case <-contEvents: + t.Fatal("Should be out of events") + default: + } // Set the task desired status to be stopped and StopContainer will be called updateSleepTask := *sleepTask @@ -978,10 +975,8 @@ func TestTaskTransitionWhenStopContainerReturnsTransientErrorBeforeSucceeding(t // StopContainer invocation should have caused it to stop eventually. contEvent = <-contEvents assert.Equal(t, contEvent.Status, api.ContainerStopped, "Expected container to be stopped") - *contEvent.SentStatus = api.ContainerStopped taskEvent = <-taskEvents assert.Equal(t, taskEvent.Status, api.TaskStopped, "Expected task to be stopped") - *taskEvent.SentStatus = api.TaskStopped select { case <-taskEvents: diff --git a/agent/vendor/github.com/docker/docker/AUTHORS b/agent/vendor/github.com/docker/docker/AUTHORS new file mode 100644 index 00000000000..246e2a33f5b --- /dev/null +++ b/agent/vendor/github.com/docker/docker/AUTHORS @@ -0,0 +1,1652 @@ +# This file lists all individuals having contributed content to the repository. +# For how it is generated, see `hack/generate-authors.sh`. + +Aanand Prasad +Aaron Davidson +Aaron Feng +Aaron Huslage +Aaron Lehmann +Aaron Welch +Abel Muiño +Abhijeet Kasurde +Abhinav Ajgaonkar +Abhishek Chanda +Abin Shahab +Adam Avilla +Adam Kunk +Adam Miller +Adam Mills +Adam Singer +Adam Walz +Aditi Rajagopal +Aditya +Adolfo Ochagavía +Adria Casas +Adrian Moisey +Adrian Mouat +Adrian Oprea +Adrien Folie +Adrien Gallouët +Ahmed Kamal +Ahmet Alp Balkan +Aidan Feldman +Aidan Hobson Sayers +AJ Bowen +Ajey Charantimath +ajneu +Akihiro Suda +Al Tobey +alambike +Alan Scherger +Alan Thompson +Albert Callarisa +Albert Zhang +Aleksa Sarai +Aleksandrs Fadins +Alena Prokharchyk +Alessandro Boch +Alessio Biancalana +Alex Chan +Alex Coventry +Alex Crawford +Alex Ellis +Alex Gaynor +Alex Olshansky +Alex Samorukov +Alex Warhawk +Alexander Artemenko +Alexander Boyd +Alexander Larsson +Alexander Morozov +Alexander Shopov +Alexandre Beslic +Alexandre González +Alexandru Sfirlogea +Alexey Guskov +Alexey Kotlyarov +Alexey Shamrin +Alexis THOMAS +Ali Dehghani +Allen Madsen +Allen Sun +almoehi +Alvaro Saurin +Alvin Richards +amangoel +Amen Belayneh +Amit Bakshi +Amit Krishnan +Amit Shukla +Amy Lindburg +Anand Patil +AnandkumarPatel +Anatoly Borodin +Anchal Agrawal +Anders Janmyr +Andre Dublin <81dublin@gmail.com> +Andre Granovsky +Andrea Luzzardi +Andrea Turli +Andreas Köhler +Andreas Savvides +Andreas Tiefenthaler +Andrei Gherzan +Andrew C. Bodine +Andrew Clay Shafer +Andrew Duckworth +Andrew France +Andrew Gerrand +Andrew Guenther +Andrew Kuklewicz +Andrew Macgregor +Andrew Macpherson +Andrew Martin +Andrew Munsell +Andrew Po +Andrew Weiss +Andrew Williams +Andrews Medina +Andrey Petrov +Andrey Stolbovsky +André Martins +andy +Andy Chambers +andy diller +Andy Goldstein +Andy Kipp +Andy Rothfusz +Andy Smith +Andy Wilson +Anes Hasicic +Anil Belur +Anil Madhavapeddy +Ankush Agarwal +Anonmily +Anthon van der Neut +Anthony Baire +Anthony Bishopric +Anthony Dahanne +Anton Löfgren +Anton Nikitin +Anton Polonskiy +Anton Tiurin +Antonio Murdaca +Antonis Kalipetis +Antony Messerli +Anuj Bahuguna +Anusha Ragunathan +apocas +ArikaChen +Arnaud Lefebvre +Arnaud Porterie +Arthur Barr +Arthur Gautier +Artur Meyster +Arun Gupta +Asbjørn Enge +averagehuman +Avi Das +Avi Miller +Avi Vaid +ayoshitake +Azat Khuyiyakhmetov +Bardia Keyoumarsi +Barnaby Gray +Barry Allard +Bartłomiej Piotrowski +Bastiaan Bakker +bdevloed +Ben Firshman +Ben Golub +Ben Hall +Ben Sargent +Ben Severson +Ben Toews +Ben Wiklund +Benjamin Atkin +Benoit Chesneau +Bernerd Schaefer +Bert Goethals +Bharath Thiruveedula +Bhiraj Butala +Bilal Amarni +Bill W +bin liu +Blake Geno +Boaz Shuster +bobby abbott +boucher +Bouke Haarsma +Boyd Hemphill +boynux +Bradley Cicenas +Bradley Wright +Brandon Liu +Brandon Philips +Brandon Rhodes +Brendan Dixon +Brent Salisbury +Brett Higgins +Brett Kochendorfer +Brian (bex) Exelbierd +Brian Bland +Brian DeHamer +Brian Dorsey +Brian Flad +Brian Goff +Brian McCallister +Brian Olsen +Brian Shumate +Brian Torres-Gil +Brian Trump +Brice Jaglin +Briehan Lombaard +Bruno Bigras +Bruno Binet +Bruno Gazzera +Bruno Renié +Bryan Bess +Bryan Boreham +Bryan Matsuo +Bryan Murphy +buddhamagnet +Burke Libbey +Byung Kang +Caleb Spare +Calen Pennington +Cameron Boehmer +Cameron Spear +Campbell Allen +Candid Dauth +Cao Weiwei +Carl Henrik Lunde +Carl Loa Odin +Carl X. Su +Carlos Alexandro Becker +Carlos Sanchez +Carol Fager-Higgins +Cary +Casey Bisson +Cedric Davies +Cezar Sa Espinola +Chad Swenson +Chance Zibolski +Chander G +Charles Chan +Charles Hooper +Charles Law +Charles Lindsay +Charles Merriam +Charles Sarrazin +Charles Smith +Charlie Lewis +Chase Bolt +ChaYoung You +Chen Chao +Chen Hanxiao +cheney90 +Chewey +Chia-liang Kao +chli +Cholerae Hu +Chris Alfonso +Chris Armstrong +Chris Dituri +Chris Fordham +Chris Khoo +Chris McKinnel +Chris Seto +Chris Snow +Chris St. Pierre +Chris Stivers +Chris Swan +Chris Wahl +Chris Weyl +chrismckinnel +Christian Berendt +Christian Böhme +Christian Persson +Christian Rotzoll +Christian Simon +Christian Stefanescu +ChristoperBiscardi +Christophe Mehay +Christophe Troestler +Christopher Currie +Christopher Jones +Christopher Latham +Christopher Rigor +Christy Perez +Chun Chen +Ciro S. Costa +Clayton Coleman +Clinton Kitson +Coenraad Loubser +Colin Dunklau +Colin Rice +Colin Walters +Collin Guarino +Colm Hally +companycy +Cory Forsyth +cressie176 +CrimsonGlory +Cristian Staretu +cristiano balducci +Cruceru Calin-Cristian +Cyril F +Daan van Berkel +Daehyeok Mun +Dafydd Crosby +dalanlan +Damian Smyth +Damien Nadé +Damien Nozay +Damjan Georgievski +Dan Anolik +Dan Buch +Dan Cotora +Dan Feldman +Dan Griffin +Dan Hirsch +Dan Keder +Dan Levy +Dan McPherson +Dan Stine +Dan Walsh +Dan Williams +Daniel Antlinger +Daniel Exner +Daniel Farrell +Daniel Garcia +Daniel Gasienica +Daniel Hiltgen +Daniel Menet +Daniel Mizyrycki +Daniel Nephin +Daniel Norberg +Daniel Nordberg +Daniel Robinson +Daniel S +Daniel Von Fange +Daniel X Moore +Daniel YC Lin +Daniel Zhang +Daniel, Dao Quang Minh +Danny Berger +Danny Yates +Darren Coxall +Darren Shepherd +Darren Stahl +Davanum Srinivas +Dave Barboza +Dave Henderson +Dave MacDonald +Dave Tucker +David Anderson +David Calavera +David Corking +David Cramer +David Currie +David Davis +David Dooling +David Gageot +David Gebler +David Lawrence +David Lechner +David M. Karr +David Mackey +David Mat +David Mcanulty +David Pelaez +David R. Jenni +David Röthlisberger +David Sheets +David Sissitka +David Trott +David Xia +David Young +Davide Ceretti +Dawn Chen +dbdd +dcylabs +decadent +deed02392 +Deng Guangxing +Deni Bertovic +Denis Gladkikh +Denis Ollier +Dennis Docter +Derek +Derek +Derek Ch +Derek McGowan +Deric Crago +Deshi Xiao +devmeyster +Devvyn Murphy +Dharmit Shah +Dieter Reuter +Dillon Dixon +Dima Stopel +Dimitri John Ledkov +Dimitris Rozakis +Dimitry Andric +Dinesh Subhraveti +Diogo Monica +DiuDiugirl +Djibril Koné +dkumor +Dmitri Logvinenko +Dmitri Shuralyov +Dmitry Demeshchuk +Dmitry Gusev +Dmitry Smirnov +Dmitry V. Krivenok +Dmitry Vorobev +Dolph Mathews +Dominik Finkbeiner +Dominik Honnef +Don Kirkby +Don Kjer +Don Spaulding +Donald Huang +Dong Chen +Donovan Jones +Doron Podoleanu +Doug Davis +Doug MacEachern +Doug Tangren +Dr Nic Williams +dragon788 +Dražen Lučanin +Drew Erny +Dustin Sallings +Ed Costello +Edmund Wagner +Eiichi Tsukata +Eike Herzbach +Eivin Giske Skaaren +Eivind Uggedal +Elan Ruusamäe +Elias Probst +Elijah Zupancic +eluck +Elvir Kuric +Emil Hernvall +Emily Maier +Emily Rose +Emir Ozer +Enguerran +Eohyung Lee +Eric Barch +Eric Hanchrow +Eric Lee +Eric Myhre +Eric Paris +Eric Rafaloff +Eric Rosenberg +Eric Sage +Eric Windisch +Eric Yang +Eric-Olivier Lamey +Erik Bray +Erik Dubbelboer +Erik Hollensbe +Erik Inge Bolsø +Erik Kristensen +Erik Weathers +Erno Hopearuoho +Erwin van der Koogh +Euan +Eugene Yakubovich +eugenkrizo +evalle +Evan Allrich +Evan Carmi +Evan Hazlett +Evan Krall +Evan Phoenix +Evan Wies +Everett Toews +Evgeny Vereshchagin +Ewa Czechowska +Eystein Måløy Stenberg +ezbercih +Fabiano Rosas +Fabio Falci +Fabio Rapposelli +Fabio Rehm +Fabrizio Regini +Fabrizio Soppelsa +Faiz Khan +falmp +Fangyuan Gao <21551127@zju.edu.cn> +Fareed Dudhia +Fathi Boudra +Federico Gimenez +Felix Geisendörfer +Felix Hupfeld +Felix Rabe +Felix Ruess +Felix Schindler +Ferenc Szabo +Fernando +Fero Volar +Ferran Rodenas +Filipe Brandenburger +Filipe Oliveira +fl0yd +Flavio Castelli +FLGMwt +Florian +Florian Klein +Florian Maier +Florian Weingarten +Florin Asavoaie +fonglh +fortinux +Francesc Campoy +Francis Chuang +Francisco Carriedo +Francisco Souza +Frank Groeneveld +Frank Herrmann +Frank Macreery +Frank Rosquin +Fred Lifton +Frederick F. Kautz IV +Frederik Loeffert +Frederik Nordahl Jul Sabroe +Freek Kalter +frosforever +fy2462 +Félix Baylac-Jacqué +Félix Cantournet +Gabe Rosenhouse +Gabor Nagy +Gabriel Monroy +GabrielNicolasAvellaneda +Galen Sampson +Gareth Rushgrove +Garrett Barboza +Gaurav +gautam, prasanna +GennadySpb +Geoffrey Bachelet +George MacRorie +George Xie +Georgi Hristozov +Gereon Frey +German DZ +Gert van Valkenhoef +Gianluca Borello +Gildas Cuisinier +gissehel +Giuseppe Mazzotta +Gleb Fotengauer-Malinovskiy +Gleb M Borisov +Glyn Normington +GoBella +Goffert van Gool +Gosuke Miyashita +Gou Rao +Govinda Fichtner +Grant Reaber +Graydon Hoare +Greg Fausak +Greg Thornton +grossws +grunny +gs11 +Guilhem Lettron +Guilherme Salgado +Guillaume Dufour +Guillaume J. Charmes +guoxiuyan +Gurjeet Singh +Guruprasad +gwx296173 +Günter Zöchbauer +Hans Kristian Flaatten +Hans Rødtang +Hao Shu Wei +Hao Zhang <21521210@zju.edu.cn> +Harald Albers +Harley Laue +Harold Cooper +Harry Zhang +He Simei +heartlock <21521209@zju.edu.cn> +Hector Castro +Henning Sprang +Hobofan +Hollie Teal +Hong Xu +hsinko <21551195@zju.edu.cn> +Hu Keping +Hu Tao +Huanzhong Zhang +Huayi Zhang +Hugo Duncan +Hugo Marisco <0x6875676f@gmail.com> +Hunter Blanks +huqun +Huu Nguyen +hyeongkyu.lee +hyp3rdino +Hyzhou <1187766782@qq.com> +Ian Babrou +Ian Bishop +Ian Bull +Ian Calvert +Ian Lee +Ian Main +Ian Truslove +Iavael +Icaro Seara +Igor Dolzhikov +Ilkka Laukkanen +Ilya Dmitrichenko +Ilya Gusev +ILYA Khlopotov +imre Fitos +inglesp +Ingo Gottwald +Isaac Dupree +Isabel Jimenez +Isao Jonas +Ivan Babrou +Ivan Fraixedes +Ivan Grcic +J Bruni +J. Nunn +Jack Danger Canty +Jacob Atzen +Jacob Edelman +Jake Champlin +Jake Moshenko +jakedt +James Allen +James Carey +James Carr +James DeFelice +James Harrison Fisher +James Kyburz +James Kyle +James Lal +James Mills +James Nugent +James Turnbull +Jamie Hannaford +Jamshid Afshar +Jan Keromnes +Jan Koprowski +Jan Pazdziora +Jan Toebes +Jan-Gerd Tenberge +Jan-Jaap Driessen +Jana Radhakrishnan +Jannick Fahlbusch +Januar Wayong +Jared Biel +Jared Hocutt +Jaroslaw Zabiello +jaseg +Jasmine Hegman +Jason Divock +Jason Giedymin +Jason Green +Jason Hall +Jason Heiss +Jason Livesay +Jason McVetta +Jason Plum +Jason Shepherd +Jason Smith +Jason Sommer +Jason Stangroome +jaxgeller +Jay +Jay +Jay Kamat +Jean-Baptiste Barth +Jean-Baptiste Dalido +Jean-Paul Calderone +Jean-Tiare Le Bigot +Jeff Anderson +Jeff Johnston +Jeff Lindsay +Jeff Mickey +Jeff Minard +Jeff Nickoloff +Jeff Silberman +Jeff Welch +Jeffrey Bolle +Jeffrey Morgan +Jeffrey van Gogh +Jenny Gebske +Jeremy Grosser +Jeremy Price +Jeremy Qian +Jeremy Unruh +Jeroen Jacobs +Jesse Dearing +Jesse Dubay +Jessica Frazelle +Jezeniel Zapanta +jgeiger +Jhon Honce +Ji.Zhilong +Jian Zhang +jianbosun +Jilles Oldenbeuving +Jim Alateras +Jim Perrin +Jimmy Cuadra +Jimmy Puckett +jimmyxian +Jinsoo Park +Jiri Popelka +Jiří Župka +jjy +jmzwcn +Joao Fernandes +Joe Beda +Joe Doliner +Joe Ferguson +Joe Gordon +Joe Shaw +Joe Van Dyk +Joel Friedly +Joel Handwell +Joel Hansson +Joel Wurtz +Joey Geiger +Joey Gibson +Joffrey F +Johan Euphrosine +Johan Rydberg +Johanan Lieberman +Johannes 'fish' Ziemke +John Costa +John Feminella +John Gardiner Myers +John Gossman +John Howard (VM) +John OBrien III +John Starks +John Tims +John Warwick +John Willis +johnharris85 +Jon Wedaman +Jonas Pfenniger +Jonathan A. Sternberg +Jonathan Boulle +Jonathan Camp +Jonathan Dowland +Jonathan Lebon +Jonathan Lomas +Jonathan McCrohan +Jonathan Mueller +Jonathan Pares +Jonathan Rudenberg +Jonathan Stoppani +Joost Cassee +Jordan +Jordan Arentsen +Jordan Sissel +Jose Diaz-Gonzalez +Joseph Anthony Pasquale Holsten +Joseph Hager +Joseph Kern +Josh +Josh Bodah +Josh Chorlton +Josh Hawn +Josh Horwitz +Josh Poimboeuf +Josiah Kiehl +José Tomás Albornoz +JP +jrabbit +Julian Taylor +Julien Barbier +Julien Bisconti +Julien Bordellier +Julien Dubois +Julien Pervillé +Julio Montes +Jun-Ru Chang +Jussi Nummelin +Justas Brazauskas +Justin Cormack +Justin Force +Justin Plock +Justin Simonelis +Justin Terry +Justyn Temme +Jyrki Puttonen +Jérôme Petazzoni +Jörg Thalheim +Kai Blin +Kai Qiang Wu(Kennan) +Kamil Domański +kamjar gerami +Kanstantsin Shautsou +Kara Alexandra +Karan Lyons +Kareem Khazem +kargakis +Karl Grzeszczak +Karol Duleba +Katie McLaughlin +Kato Kazuyoshi +Katrina Owen +Kawsar Saiyeed +kayrus +Ke Xu +Keith Hudgins +Keli Hu +Ken Cochrane +Ken Herner +Ken ICHIKAWA +Kenfe-Mickaël Laventure +Kenjiro Nakayama +Kent Johnson +Kevin "qwazerty" Houdebert +Kevin Burke +Kevin Clark +Kevin J. Lynagh +Kevin Jing Qiu +Kevin Menard +Kevin P. Kucharczyk +Kevin Richardson +Kevin Shi +Kevin Wallace +Kevin Yap +kevinmeredith +Keyvan Fatehi +kies +Kim BKC Carlbacker +Kim Eik +Kimbro Staken +Kir Kolyshkin +Kiran Gangadharan +Kirill Kolyshkin +Kirill SIbirev +knappe +Kohei Tsuruta +Koichi Shiraishi +Konrad Kleine +Konstantin L +Konstantin Pelykh +Krasimir Georgiev +Kris-Mikael Krister +Kristian Haugene +Kristina Zabunova +krrg +Kun Zhang +Kunal Kushwaha +Kyle Conroy +Kyle Linden +kyu +Lachlan Coote +Lai Jiangshan +Lajos Papp +Lakshan Perera +Lalatendu Mohanty +lalyos +Lance Chen +Lance Kinley +Lars Butler +Lars Kellogg-Stedman +Lars R. Damerow +Laszlo Meszaros +Laurent Erignoux +Laurie Voss +Leandro Siqueira +Lee Chao <932819864@qq.com> +Lee, Meng-Han +leeplay +Lei Jitang +Len Weincier +Lennie +Leszek Kowalski +Levi Blackstone +Levi Gross +Lewis Marshall +Lewis Peckover +Liam Macgillavry +Liana Lo +Liang Mingqiang +Liang-Chi Hsieh +liaoqingwei +limsy +Lin Lu +LingFaKe +Linus Heckemann +Liran Tal +Liron Levin +Liu Bo +Liu Hua +lixiaobing10051267 +LIZAO LI +Lloyd Dewolf +Lokesh Mandvekar +longliqiang88 <394564827@qq.com> +Lorenz Leutgeb +Lorenzo Fontana +Louis Opter +Luca Marturana +Luca Orlandi +Luca-Bogdan Grigorescu +Lucas Chan +Lucas Chi +Luciano Mores +Luis Martínez de Bartolomé Izquierdo +Lukas Waslowski +lukaspustina +Lukasz Zajaczkowski +lukemarsden +Lynda O'Leary +Lénaïc Huard +Ma Shimiao +Mabin +Madhav Puri +Madhu Venugopal +Mageee <21521230.zju.edu.cn> +Mahesh Tiyyagura +malnick +Malte Janduda +manchoz +Manfred Touron +Manfred Zabarauskas +Mansi Nahar +mansinahar +Manuel Meurer +Manuel Woelker +mapk0y +Marc Abramowitz +Marc Kuo +Marc Tamsky +Marcelo Salazar +Marco Hennings +Marcus Farkas +Marcus Linke +Marcus Ramberg +Marek Goldmann +Marian Marinov +Marianna Tessel +Mario Loriedo +Marius Gundersen +Marius Sturm +Marius Voila +Mark Allen +Mark McGranaghan +Mark McKinstry +Mark West +Marko Mikulicic +Marko Tibold +Markus Fix +Martijn Dwars +Martijn van Oosterhout +Martin Honermeyer +Martin Kelly +Martin Mosegaard Amdisen +Martin Redmond +Mary Anthony +Masahito Zembutsu +Mason Malone +Mateusz Sulima +Mathias Monnerville +Mathieu Le Marec - Pasquet +Matt Apperson +Matt Bachmann +Matt Bentley +Matt Haggard +Matt Hoyle +Matt McCormick +Matt Moore +Matt Richardson +Matt Robenolt +Matthew Heon +Matthew Mayer +Matthew Mueller +Matthew Riley +Matthias Klumpp +Matthias Kühnle +Matthias Rampke +Matthieu Hauglustaine +mattymo +mattyw +Mauricio Garavaglia +mauriyouth +Max Shytikov +Maxim Fedchyshyn +Maxim Ivanov +Maxim Kulkin +Maxim Treskin +Maxime Petazzoni +Meaglith Ma +meejah +Megan Kostick +Mehul Kar +Mei ChunTao +Mengdi Gao +Mert Yazıcıoğlu +mgniu +Micah Zoltu +Michael A. Smith +Michael Bridgen +Michael Brown +Michael Chiang +Michael Crosby +Michael Currie +Michael Friis +Michael Gorsuch +Michael Grauer +Michael Holzheu +Michael Hudson-Doyle +Michael Huettermann +Michael Käufl +Michael Neale +Michael Prokop +Michael Scharf +Michael Stapelberg +Michael Steinert +Michael Thies +Michael West +Michal Fojtik +Michal Gebauer +Michal Jemala +Michal Minar +Michal Wieczorek +Michaël Pailloncy +Michał Czeraszkiewicz +Michiel@unhosted +Mickaël FORTUNATO +Miguel Angel Fernández +Miguel Morales +Mihai Borobocea +Mihuleacc Sergiu +Mike Brown +Mike Chelen +Mike Danese +Mike Dillon +Mike Dougherty +Mike Gaffney +Mike Goelzer +Mike Leone +Mike MacCana +Mike Naberezny +Mike Snitzer +mikelinjie <294893458@qq.com> +Mikhail Sobolev +Miloslav Trmač +mingqing +Mingzhen Feng +Misty Stanley-Jones +Mitch Capper +mlarcher +Mohammad Banikazemi +Mohammed Aaqib Ansari +Mohit Soni +Morgan Bauer +Morgante Pell +Morgy93 +Morten Siebuhr +Morton Fox +Moysés Borges +mqliang +Mrunal Patel +msabansal +mschurenko +muge +Mustafa Akın +Muthukumar R +Máximo Cuadros +Médi-Rémi Hashim +Nahum Shalman +Nakul Pathak +Nalin Dahyabhai +Nan Monnand Deng +Naoki Orii +Natalie Parker +Natanael Copa +Nate Brennand +Nate Eagleson +Nate Jones +Nathan Hsieh +Nathan Kleyn +Nathan LeClaire +Nathan McCauley +Nathan Williams +Neal McBurnett +Neil Peterson +Nelson Chen +Neyazul Haque +Nghia Tran +Niall O'Higgins +Nicholas E. Rabenau +nick +Nick DeCoursin +Nick Irvine +Nick Parker +Nick Payne +Nick Stenning +Nick Stinemates +Nicola Kabar +Nicolas Borboën +Nicolas De loof +Nicolas Dudebout +Nicolas Goy +Nicolas Kaiser +Nicolás Hock Isaza +Nigel Poulton +NikolaMandic +nikolas +Nirmal Mehta +Nishant Totla +NIWA Hideyuki +noducks +Nolan Darilek +nponeccop +Nuutti Kotivuori +nzwsch +O.S. Tezer +objectified +OddBloke +odk- +Oguz Bilgic +Oh Jinkyun +Ohad Schneider +ohmystack +Ole Reifschneider +Oliver Neal +Olivier Gambier +Olle Jonsson +Oriol Francès +orkaa +Oskar Niburski +Otto Kekäläinen +oyld +ozlerhakan +paetling +pandrew +panticz +Paolo G. Giarrusso +Pascal Borreli +Pascal Hartig +Patrick Böänziger +Patrick Devine +Patrick Hemmer +Patrick Stapleton +pattichen +Paul +paul +Paul Annesley +Paul Bellamy +Paul Bowsher +Paul Furtado +Paul Hammond +Paul Jimenez +Paul Lietar +Paul Liljenberg +Paul Morie +Paul Nasrat +Paul Weaver +Paulo Ribeiro +Pavel Lobashov +Pavel Pospisil +Pavel Sutyrin +Pavel Tikhomirov +Pavlos Ratis +Pavol Vargovcik +Peeyush Gupta +Peggy Li +Pei Su +Penghan Wang +perhapszzy@sina.com +pestophagous +Peter Bourgon +Peter Braden +Peter Choi +Peter Dave Hello +Peter Edge +Peter Ericson +Peter Esbensen +Peter Malmgren +Peter Salvatore +Peter Volpe +Peter Waller +Petr Švihlík +Phil +Phil Estes +Phil Spitler +Philip Monroe +Philipp Wahala +Philipp Weissensteiner +Phillip Alexander +pidster +Piergiuliano Bossi +Pierre +Pierre Carrier +Pierre Dal-Pra +Pierre Wacrenier +Pierre-Alain RIVIERE +Piotr Bogdan +pixelistik +Porjo +Poul Kjeldager Sørensen +Pradeep Chhetri +Prasanna Gautam +Prayag Verma +Przemek Hejman +pysqz +qg <1373319223@qq.com> +qhuang +Qiang Huang +qq690388648 <690388648@qq.com> +Quentin Brossard +Quentin Perez +Quentin Tayssier +r0n22 +Rafal Jeczalik +Rafe Colton +Raghavendra K T +Raghuram Devarakonda +Rajat Pandit +Rajdeep Dua +Ralf Sippl +Ralle +Ralph Bean +Ramkumar Ramachandra +Ramon Brooker +Ramon van Alteren +Ray Tsang +ReadmeCritic +Recursive Madman +Regan McCooey +Remi Rampin +Renato Riccieri Santos Zannon +resouer +rgstephens +Rhys Hiltner +Rich Moyse +Rich Seymour +Richard +Richard Burnison +Richard Harvey +Richard Mathie +Richard Metzler +Richard Scothern +Richo Healey +Rick Bradley +Rick van de Loo +Rick Wieman +Rik Nijessen +Riku Voipio +Riley Guerin +Ritesh H Shukla +Riyaz Faizullabhoy +Rob Vesse +Robert Bachmann +Robert Bittle +Robert Obryk +Robert Stern +Robert Terhaar +Robert Wallis +Roberto G. Hashioka +Robin Naundorf +Robin Schneider +Robin Speekenbrink +robpc +Rodolfo Carvalho +Rodrigo Vaz +Roel Van Nyen +Roger Peppe +Rohit Jnagal +Rohit Kadam +Roland Huß +Roland Kammerer +Roland Moriz +Roma Sokolov +Roman Strashkin +Ron Smits +Ron Williams +root +root +root +root +root +Rory Hunter +Rory McCune +Ross Boucher +Rovanion Luckey +Rozhnov Alexandr +rsmoorthy +Rudolph Gottesheim +Rui Lopes +Runshen Zhu +Ryan Anderson +Ryan Aslett +Ryan Belgrave +Ryan Detzel +Ryan Fowler +Ryan McLaughlin +Ryan O'Donnell +Ryan Seto +Ryan Thomas +Ryan Trauntvein +Ryan Wallner +RyanDeng +Rémy Greinhofer +s. rannou +s00318865 +Sabin Basyal +Sachin Joshi +Sagar Hani +Sainath Grandhi +sakeven +Sally O'Malley +Sam Abed +Sam Alba +Sam Bailey +Sam J Sharpe +Sam Neirinck +Sam Reis +Sam Rijs +Sambuddha Basu +Sami Wagiaalla +Samuel Andaya +Samuel Dion-Girardeau +Samuel Karp +Samuel PHAN +Sankar சங்கர் +Sanket Saurav +Santhosh Manohar +sapphiredev +Satnam Singh +satoru +Satoshi Amemiya +Satoshi Tagomori +scaleoutsean +Scott Bessler +Scott Collier +Scott Johnston +Scott Stamp +Scott Walls +sdreyesg +Sean Christopherson +Sean Cronin +Sean OMeara +Sean P. Kane +Sebastiaan van Steenis +Sebastiaan van Stijn +Senthil Kumar Selvaraj +Senthil Kumaran +SeongJae Park +Seongyeol Lim +Serge Hallyn +Sergey Alekseev +Sergey Evstifeev +Serhat Gülçiçek +Sevki Hasirci +Shane Canon +Shane da Silva +shaunol +Shawn Landden +Shawn Siefkas +shawnhe +Shekhar Gulati +Sheng Yang +Shengbo Song +Shev Yan +Shih-Yuan Lee +Shijiang Wei +Shishir Mahajan +Shoubhik Bose +Shourya Sarcar +shuai-z +Shukui Yang +Shuwei Hao +Sian Lerk Lau +sidharthamani +Silas Sewell +Simei He +Simon Eskildsen +Simon Leinen +Simon Taranto +Sindhu S +Sjoerd Langkemper +skaasten +Solganik Alexander +Solomon Hykes +Song Gao +Soshi Katsuta +Soulou +Spencer Brown +Spencer Smith +Sridatta Thatipamala +Sridhar Ratnakumar +Srini Brahmaroutu +srinsriv +Steeve Morin +Stefan Berger +Stefan J. Wernli +Stefan Praszalowicz +Stefan Scherer +Stefan Staudenmeyer +Stefan Weil +Stephen Crosby +Stephen Day +Stephen Drake +Stephen Rust +Steve Durrheimer +Steve Francia +Steve Koch +Steven Burgess +Steven Erenst +Steven Iveson +Steven Merrill +Steven Richards +Steven Taylor +Subhajit Ghosh +Sujith Haridasan +Suryakumar Sudar +Sven Dowideit +Swapnil Daingade +Sylvain Baubeau +Sylvain Bellemare +Sébastien +Sébastien Luttringer +Sébastien Stormacq +Tadej Janež +TAGOMORI Satoshi +tang0th +Tangi COLIN +Tatsuki Sugiura +Tatsushi Inagaki +Taylor Jones +tbonza +Ted M. Young +Tehmasp Chaudhri +Tejesh Mehta +terryding77 <550147740@qq.com> +tgic +Thatcher Peskens +theadactyl +Thell 'Bo' Fowler +Thermionix +Thijs Terlouw +Thomas Bikeev +Thomas Frössman +Thomas Gazagnaire +Thomas Grainger +Thomas Hansen +Thomas Leonard +Thomas LEVEIL +Thomas Orozco +Thomas Riccardi +Thomas Schroeter +Thomas Sjögren +Thomas Swift +Thomas Tanaka +Thomas Texier +Tianon Gravi +Tianyi Wang +Tibor Vass +Tiffany Jernigan +Tiffany Low +Tim Bosse +Tim Dettrick +Tim Düsterhus +Tim Hockin +Tim Ruffles +Tim Smith +Tim Terhorst +Tim Wang +Tim Waugh +Tim Wraight +timfeirg +Timothy Hobbs +tjwebb123 +tobe +Tobias Bieniek +Tobias Bradtke +Tobias Gesellchen +Tobias Klauser +Tobias Munk +Tobias Schmidt +Tobias Schwab +Todd Crane +Todd Lunter +Todd Whiteman +Toli Kuznets +Tom Barlow +Tom Denham +Tom Fotherby +Tom Howe +Tom Hulihan +Tom Maaswinkel +Tom X. Tobin +Tomas Tomecek +Tomasz Kopczynski +Tomasz Lipinski +Tomasz Nurkiewicz +Tommaso Visconti +Tomáš Hrčka +Tonis Tiigi +Tonny Xu +Tony Daws +Tony Miller +toogley +Torstein Husebø +tpng +tracylihui <793912329@qq.com> +Travis Cline +Travis Thieman +Trent Ogren +Trevor +Trevor Pounds +trishnaguha +Tristan Carel +Troy Denton +Tyler Brock +Tzu-Jung Lee +Tõnis Tiigi +Ulysse Carion +unknown +vagrant +Vaidas Jablonskis +Veres Lajos +vgeta +Victor Algaze +Victor Coisne +Victor Costan +Victor I. Wood +Victor Lyuboslavsky +Victor Marmol +Victor Palma +Victor Vieux +Victoria Bialas +Vijaya Kumar K +Viktor Stanchev +Viktor Vojnovski +VinayRaghavanKS +Vincent Batts +Vincent Bernat +Vincent Bernat +Vincent Demeester +Vincent Giersch +Vincent Mayers +Vincent Woo +Vinod Kulkarni +Vishal Doshi +Vishnu Kannan +Vitor Monteiro +Vivek Agarwal +Vivek Dasgupta +Vivek Goyal +Vladimir Bulyga +Vladimir Kirillov +Vladimir Pouzanov +Vladimir Rutsky +Vladimir Varankin +VladimirAus +Vojtech Vitek (V-Teq) +waitingkuo +Walter Leibbrandt +Walter Stanish +WANG Chao +Wang Xing +Ward Vandewege +WarheadsSE +Wayne Chang +Wei-Ting Kuo +weiyan +Weiyang Zhu +Wen Cheng Ma +Wendel Fleming +Wenkai Yin +Wenxuan Zhao +Wenyu You <21551128@zju.edu.cn> +Wes Morgan +Will Dietz +Will Rouesnel +Will Weaver +willhf +William Delanoue +William Henry +William Hubbs +William Riancho +William Thurston +WiseTrem +wlan0 +Wolfgang Powisch +wonderflow +Wonjun Kim +xamyzhao +Xianlu Bird +XiaoBing Jiang +Xiaoxu Chen +xiekeyang +Xinzi Zhou +Xiuming Chen +xlgao-zju +xuzhaokui +Yahya +YAMADA Tsuyoshi +Yan Feng +Yang Bai +yangshukui +Yanqiang Miao +Yasunori Mahata +Yestin Sun +Yi EungJun +Yibai Zhang +Yihang Ho +Ying Li +Yohei Ueda +Yong Tang +Yongzhi Pan +yorkie +Youcef YEKHLEF +Yuan Sun +yuchangchun +yuchengxia +yuexiao-wang +YuPengZTE +Yurii Rashkovskii +yuzou +Zac Dover +Zach Borboa +Zachary Jaffee +Zain Memon +Zaiste! +Zane DeGraffenried +Zefan Li +Zen Lin(Zhinan Lin) +Zhang Kun +Zhang Wei +Zhang Wentao +Zhenan Ye <21551168@zju.edu.cn> +zhouhao +Zhu Guihua +Zhuoyun Wei +Zilin Du +zimbatm +Ziming Dong +ZJUshuaizhou <21551191@zju.edu.cn> +zmarouf +Zoltan Tombol +zqh +Zuhayr Elahi +Zunayed Ali +Álex González +Álvaro Lázaro +Átila Camurça Alves +尹吉峰 +搏通 diff --git a/agent/vendor/github.com/docker/docker/project/CONTRIBUTORS.md b/agent/vendor/github.com/docker/docker/project/CONTRIBUTORS.md new file mode 120000 index 00000000000..44fcc634393 --- /dev/null +++ b/agent/vendor/github.com/docker/docker/project/CONTRIBUTORS.md @@ -0,0 +1 @@ +../CONTRIBUTING.md \ No newline at end of file diff --git a/agent/vendor/github.com/golang/mock/AUTHORS b/agent/vendor/github.com/golang/mock/AUTHORS new file mode 100644 index 00000000000..660b8ccc8ae --- /dev/null +++ b/agent/vendor/github.com/golang/mock/AUTHORS @@ -0,0 +1,12 @@ +# This is the official list of GoMock authors for copyright purposes. +# This file is distinct from the CONTRIBUTORS files. +# See the latter for an explanation. + +# Names should be added to this file as +# Name or Organization +# The email address is not required for organizations. + +# Please keep the list sorted. + +Alex Reece +Google Inc. diff --git a/agent/vendor/github.com/golang/mock/CONTRIBUTORS b/agent/vendor/github.com/golang/mock/CONTRIBUTORS new file mode 100644 index 00000000000..def849cab1b --- /dev/null +++ b/agent/vendor/github.com/golang/mock/CONTRIBUTORS @@ -0,0 +1,37 @@ +# This is the official list of people who can contribute (and typically +# have contributed) code to the gomock repository. +# The AUTHORS file lists the copyright holders; this file +# lists people. For example, Google employees are listed here +# but not in AUTHORS, because Google holds the copyright. +# +# The submission process automatically checks to make sure +# that people submitting code are listed in this file (by email address). +# +# Names should be added to this file only after verifying that +# the individual or the individual's organization has agreed to +# the appropriate Contributor License Agreement, found here: +# +# http://code.google.com/legal/individual-cla-v1.0.html +# http://code.google.com/legal/corporate-cla-v1.0.html +# +# The agreement for individuals can be filled out on the web. +# +# When adding J Random Contributor's name to this file, +# either J's name or J's organization's name should be +# added to the AUTHORS file, depending on whether the +# individual or corporate CLA was used. + +# Names should be added to this file like so: +# Name +# +# An entry with two email addresses specifies that the +# first address should be used in the submit logs and +# that the second address should be recognized as the +# same person when interacting with Rietveld. + +# Please keep the list sorted. + +Aaron Jacobs +Alex Reece +David Symonds +Ryan Barrett diff --git a/agent/vendor/github.com/golang/mock/gomock/call.go b/agent/vendor/github.com/golang/mock/gomock/call.go index ea1e09227e2..c5601970e1e 100644 --- a/agent/vendor/github.com/golang/mock/gomock/call.go +++ b/agent/vendor/github.com/golang/mock/gomock/call.go @@ -41,11 +41,32 @@ type Call struct { setArgs map[int]reflect.Value } +// AnyTimes allows the expectation to be called 0 or more times func (c *Call) AnyTimes() *Call { c.minCalls, c.maxCalls = 0, 1e8 // close enough to infinity return c } +// MinTimes requires the call to occur at least n times. If AnyTimes or MaxTimes have not been called, MinTimes also +// sets the maximum number of calls to infinity. +func (c *Call) MinTimes(n int) *Call { + c.minCalls = n + if c.maxCalls == 1 { + c.maxCalls = 1e8 + } + return c +} + +// MaxTimes limits the number of calls to n times. If AnyTimes or MinTimes have not been called, MaxTimes also +// sets the minimum number of calls to 0. +func (c *Call) MaxTimes(n int) *Call { + c.maxCalls = n + if c.minCalls == 1 { + c.minCalls = 0 + } + return c +} + // Do declares the action to run when the call is matched. // It takes an interface{} argument to support n-arity functions. func (c *Call) Do(f interface{}) *Call { diff --git a/agent/vendor/golang.org/x/net/AUTHORS b/agent/vendor/golang.org/x/net/AUTHORS new file mode 100644 index 00000000000..15167cd746c --- /dev/null +++ b/agent/vendor/golang.org/x/net/AUTHORS @@ -0,0 +1,3 @@ +# This source code refers to The Go Authors for copyright purposes. +# The master list of authors is in the main Go distribution, +# visible at http://tip.golang.org/AUTHORS. diff --git a/agent/vendor/golang.org/x/net/CONTRIBUTORS b/agent/vendor/golang.org/x/net/CONTRIBUTORS new file mode 100644 index 00000000000..1c4577e9680 --- /dev/null +++ b/agent/vendor/golang.org/x/net/CONTRIBUTORS @@ -0,0 +1,3 @@ +# This source code was written by the Go contributors. +# The master list of contributors is in the main Go distribution, +# visible at http://tip.golang.org/CONTRIBUTORS. diff --git a/agent/vendor/golang.org/x/sys/AUTHORS b/agent/vendor/golang.org/x/sys/AUTHORS new file mode 100644 index 00000000000..15167cd746c --- /dev/null +++ b/agent/vendor/golang.org/x/sys/AUTHORS @@ -0,0 +1,3 @@ +# This source code refers to The Go Authors for copyright purposes. +# The master list of authors is in the main Go distribution, +# visible at http://tip.golang.org/AUTHORS. diff --git a/agent/vendor/golang.org/x/sys/CONTRIBUTORS b/agent/vendor/golang.org/x/sys/CONTRIBUTORS new file mode 100644 index 00000000000..1c4577e9680 --- /dev/null +++ b/agent/vendor/golang.org/x/sys/CONTRIBUTORS @@ -0,0 +1,3 @@ +# This source code was written by the Go contributors. +# The master list of contributors is in the main Go distribution, +# visible at http://tip.golang.org/CONTRIBUTORS. diff --git a/agent/vendor/golang.org/x/tools/AUTHORS b/agent/vendor/golang.org/x/tools/AUTHORS new file mode 100644 index 00000000000..15167cd746c --- /dev/null +++ b/agent/vendor/golang.org/x/tools/AUTHORS @@ -0,0 +1,3 @@ +# This source code refers to The Go Authors for copyright purposes. +# The master list of authors is in the main Go distribution, +# visible at http://tip.golang.org/AUTHORS. diff --git a/agent/vendor/golang.org/x/tools/CONTRIBUTORS b/agent/vendor/golang.org/x/tools/CONTRIBUTORS new file mode 100644 index 00000000000..1c4577e9680 --- /dev/null +++ b/agent/vendor/golang.org/x/tools/CONTRIBUTORS @@ -0,0 +1,3 @@ +# This source code was written by the Go contributors. +# The master list of contributors is in the main Go distribution, +# visible at http://tip.golang.org/CONTRIBUTORS. From 1be54202b3fcbe95ab6935d178e354849c1e4c11 Mon Sep 17 00:00:00 2001 From: liwen wu Date: Wed, 21 Dec 2016 00:12:45 +0000 Subject: [PATCH 02/27] Fixed https://github.com/aws/amazon-ecs-agent/issues/506 --- agent/acs/handler/acs_handler_test.go | 8 +++- agent/engine/docker_container_engine.go | 6 +-- agent/stats/container.go | 3 +- agent/stats/engine.go | 14 ++++++- agent/stats/queue.go | 27 +++++++++--- agent/stats/queue_test.go | 55 ++++++++++++++++++++++++- agent/tcs/client/client.go | 19 ++++++--- agent/tcs/client/client_test.go | 17 +++++++- 8 files changed, 128 insertions(+), 21 deletions(-) diff --git a/agent/acs/handler/acs_handler_test.go b/agent/acs/handler/acs_handler_test.go index 8db2d11ab5f..58b7b08488b 100644 --- a/agent/acs/handler/acs_handler_test.go +++ b/agent/acs/handler/acs_handler_test.go @@ -1,4 +1,4 @@ -// Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the @@ -518,9 +518,13 @@ func TestHandlerReconnectDelayForInactiveInstanceError(t *testing.T) { }).Return(fmt.Errorf("InactiveInstanceException:")), mockWsClient.EXPECT().Connect().Do(func() { reconnectDelay := time.Now().Sub(firstConnectionAttemptTime) + reconnectDelayTime := time.Now() t.Logf("Delay between successive connections: %v", reconnectDelay) + timeSubFuncSlopAllowed := 2 * time.Millisecond if reconnectDelay < inactiveInstanceReconnectDelay { - t.Errorf("Reconnect delay %v is not less than inactive instance reconnect delay %v", reconnectDelay, inactiveInstanceReconnectDelay) + // On windows platform, we found issue with time.Now().Sub(...) reporting 199.9989 even + // after the code has already waited for time.NewTimer(200)ms. + assert.WithinDuration(t, reconnectDelayTime, firstConnectionAttemptTime.Add(inactiveInstanceReconnectDelay), timeSubFuncSlopAllowed) } cancel() }).Return(io.EOF), diff --git a/agent/engine/docker_container_engine.go b/agent/engine/docker_container_engine.go index d0de6fde89c..10548f5d9c7 100644 --- a/agent/engine/docker_container_engine.go +++ b/agent/engine/docker_container_engine.go @@ -61,9 +61,9 @@ const ( // output will be suppressed in debug mode pullStatusSuppressDelay = 2 * time.Second - // statsInactivityTimeout controls the amount of time we hold open a + // StatsInactivityTimeout controls the amount of time we hold open a // connection to the Docker daemon waiting for stats data - statsInactivityTimeout = 5 * time.Second + StatsInactivityTimeout = 5 * time.Second ) // DockerClient interface to make testing it easier @@ -809,7 +809,7 @@ func (dg *dockerGoClient) Stats(id string, ctx context.Context) (<-chan *docker. Stats: stats, Stream: true, Context: ctx, - InactivityTimeout: statsInactivityTimeout, + InactivityTimeout: StatsInactivityTimeout, } go func() { diff --git a/agent/stats/container.go b/agent/stats/container.go index 6ca6ba6d94f..c3c467d8e82 100644 --- a/agent/stats/container.go +++ b/agent/stats/container.go @@ -1,4 +1,4 @@ -// Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the @@ -47,6 +47,7 @@ func newStatsContainer(dockerID string, client ecsengine.DockerClient, resolver func (container *StatsContainer) StartStatsCollection() { // Create the queue to store utilization data from docker stats container.statsQueue = NewQueue(ContainerStatsBufferLength) + container.statsQueue.Reset() go container.collect() } diff --git a/agent/stats/engine.go b/agent/stats/engine.go index af7d42935a6..d2f73840c5f 100644 --- a/agent/stats/engine.go +++ b/agent/stats/engine.go @@ -1,4 +1,4 @@ -// Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the @@ -16,6 +16,7 @@ package stats //go:generate go run ../../scripts/generate/mockgen.go github.com/aws/amazon-ecs-agent/agent/stats Engine mock/$GOFILE import ( + "errors" "fmt" "sync" "time" @@ -35,6 +36,7 @@ import ( const ( containerChangeHandler = "DockerStatsEngineDockerEventsHandler" listContainersTimeout = 10 * time.Minute + queueResetThreshold = 2 * ecsengine.StatsInactivityTimeout ) // DockerContainerMetadataResolver implements ContainerMetadataResolver for @@ -67,6 +69,7 @@ type DockerStatsEngine struct { // dockerStatsEngine is a singleton object of DockerStatsEngine. // TODO make dockerStatsEngine not a singleton object var dockerStatsEngine *DockerStatsEngine +var EmptyMetricsError = errors.New("No task metrics to report") // ResolveTask resolves the api task object, given container id. func (resolver *DockerContainerMetadataResolver) ResolveTask(dockerID string) (*api.Task, error) { @@ -234,7 +237,7 @@ func (engine *DockerStatsEngine) GetInstanceMetrics() (*ecstcs.MetricsMetadata, if len(taskMetrics) == 0 { // Not idle. Expect taskMetrics to be there. - return nil, nil, fmt.Errorf("No task metrics to report") + return nil, nil, EmptyMetricsError } // Reset current stats. Retaining older stats results in incorrect utilization stats @@ -398,6 +401,13 @@ func (engine *DockerStatsEngine) getContainerMetricsForTask(taskArn string) ([]* engine.doRemoveContainer(container, taskArn) continue } + + if !container.statsQueue.enoughDatapointsInBuffer() && + !container.statsQueue.resetThresholdElapsed(queueResetThreshold) { + seelog.Debugf("Stats not ready for container %s", dockerID) + continue + } + // Container is not terminal. Get CPU stats set. cpuStatsSet, err := container.statsQueue.GetCPUStatsSet() if err != nil { diff --git a/agent/stats/queue.go b/agent/stats/queue.go index d02a571c64d..f94fc01b599 100644 --- a/agent/stats/queue.go +++ b/agent/stats/queue.go @@ -1,4 +1,4 @@ -// Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the @@ -17,6 +17,7 @@ import ( "fmt" "math" "sync" + "time" "github.com/aws/amazon-ecs-agent/agent/tcs/model/ecstcs" "github.com/cihub/seelog" @@ -27,11 +28,14 @@ const ( BytesInMiB = 1024 * 1024 ) +const minimumQueueDatapoints = 2 + // Queue abstracts a queue using UsageStats slice. type Queue struct { - buffer []UsageStats - maxSize int - bufferLock sync.RWMutex + buffer []UsageStats + maxSize int + lastResetTime time.Time + bufferLock sync.RWMutex } // NewQueue creates a queue. @@ -46,7 +50,7 @@ func NewQueue(maxSize int) *Queue { func (queue *Queue) Reset() { queue.bufferLock.Lock() defer queue.bufferLock.Unlock() - + queue.lastResetTime = time.Now() queue.buffer = queue.buffer[:0] } @@ -133,6 +137,19 @@ func getMemoryUsagePerc(s *UsageStats) float64 { type getUsageFunc func(*UsageStats) float64 +func (queue *Queue) resetThresholdElapsed(timeout time.Duration) bool { + queue.bufferLock.RLock() + defer queue.bufferLock.RUnlock() + duration := time.Since(queue.lastResetTime) + return duration.Seconds() > timeout.Seconds() +} + +func (queue *Queue) enoughDatapointsInBuffer() bool { + queue.bufferLock.RLock() + defer queue.bufferLock.RUnlock() + return len(queue.buffer) >= minimumQueueDatapoints +} + // getCWStatsSet gets the stats set for either CPU or Memory based on the // function pointer. func (queue *Queue) getCWStatsSet(f getUsageFunc) (*ecstcs.CWStatsSet, error) { diff --git a/agent/stats/queue_test.go b/agent/stats/queue_test.go index af705ab8fe1..5dbfe2352c7 100644 --- a/agent/stats/queue_test.go +++ b/agent/stats/queue_test.go @@ -1,5 +1,5 @@ //+build !integration -// Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the @@ -20,6 +20,7 @@ import ( "time" "github.com/aws/aws-sdk-go/aws" + "github.com/stretchr/testify/assert" ) const ( @@ -309,3 +310,55 @@ func TestCpuStatsSetNotSetToInfinity(t *testing.T) { t.Errorf("Computed cpuStatsSet.SampleCount (%d) != expected value (%d)", sampleCount, 1) } } + +func TestResetThresholdElapsed(t *testing.T) { + // create a queue + queueLength := 3 + queue := NewQueue(queueLength) + + queue.Reset() + + thresholdElapsed := queue.resetThresholdElapsed(2 * time.Millisecond) + assert.False(t, thresholdElapsed, "Queue reset threshold is not expected to elapse right after reset") + + time.Sleep(3 * time.Millisecond) + thresholdElapsed = queue.resetThresholdElapsed(2 * time.Millisecond) + + assert.True(t, thresholdElapsed, "Queue reset threshold is expected to elapse after waiting") +} + +func TestEnoughDatapointsInBuffer(t *testing.T) { + // timestamps will be used to simulate +Inf CPU Usage + // timestamps[0] = timestamps[1] + timestamps := []time.Time{ + parseNanoTime("2015-02-12T21:22:05.131117533Z"), + parseNanoTime("2015-02-12T21:22:05.131117533Z"), + parseNanoTime("2015-02-12T21:22:05.333776335Z"), + } + cpuTimes := []uint64{ + 22400432, + 116499979, + 248503503, + } + memoryUtilizationInBytes := []uint64{ + 3649536, + 3649536, + 3649536, + } + // create a queue + queueLength := 3 + queue := NewQueue(queueLength) + + enoughDataPoints := queue.enoughDatapointsInBuffer() + assert.False(t, enoughDataPoints, "Queue is expected to not have enough data points right after creation") + for i, time := range timestamps { + queue.Add(&ContainerStats{cpuUsage: cpuTimes[i], memoryUsage: memoryUtilizationInBytes[i], timestamp: time}) + } + + enoughDataPoints = queue.enoughDatapointsInBuffer() + assert.True(t, enoughDataPoints, "Queue is expected to have enough data points when it has more than 2 msgs queued") + + queue.Reset() + enoughDataPoints = queue.enoughDatapointsInBuffer() + assert.False(t, enoughDataPoints, "Queue is expected to not have enough data points right after RESET") +} diff --git a/agent/tcs/client/client.go b/agent/tcs/client/client.go index 1bb73aea1f6..c869fb121a7 100644 --- a/agent/tcs/client/client.go +++ b/agent/tcs/client/client.go @@ -1,4 +1,4 @@ -// Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the @@ -137,12 +137,18 @@ func (cs *clientServer) publishMetrics() { // Publish metrics immediately after we connect and wait for ticks. This makes // sure that there is no data loss when a scheduled metrics publishing fails // due to a connection reset. - cs.publishMetricsOnce() + err := cs.publishMetricsOnce() + if err != nil && err != stats.EmptyMetricsError { + seelog.Warnf("Error publishing metrics: %v", err) + } // don't simply range over the ticker since its channel doesn't ever get closed for { select { case <-cs.publishTicker.C: - cs.publishMetricsOnce() + err := cs.publishMetricsOnce() + if err != nil { + seelog.Warnf("Error publishing metrics: %v", err) + } case <-cs.endPublish: return } @@ -150,20 +156,21 @@ func (cs *clientServer) publishMetrics() { } // publishMetricsOnce is invoked by the ticker to periodically publish metrics to backend. -func (cs *clientServer) publishMetricsOnce() { +func (cs *clientServer) publishMetricsOnce() error { // Get the list of objects to send to backend. requests, err := cs.metricsToPublishMetricRequests() if err != nil { - seelog.Warnf("Error getting instance metrics: %v", err) + return err } // Make the publish metrics request to the backend. for _, request := range requests { err = cs.MakeRequest(request) if err != nil { - seelog.Warnf("Error publishing metrics: %v. Request: %v", err, request) + return err } } + return nil } // metricsToPublishMetricRequests gets task metrics and converts them to a list of PublishMetricRequest diff --git a/agent/tcs/client/client_test.go b/agent/tcs/client/client_test.go index b31423996f6..7d1639c450a 100644 --- a/agent/tcs/client/client_test.go +++ b/agent/tcs/client/client_test.go @@ -1,4 +1,4 @@ -// Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the @@ -32,6 +32,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/gorilla/websocket" + "github.com/stretchr/testify/assert" "golang.org/x/net/context" ) @@ -79,6 +80,12 @@ func (engine *mockStatsEngine) GetInstanceMetrics() (*ecstcs.MetricsMetadata, [] return nil, nil, fmt.Errorf("uninitialized") } +type emptyStatsEngine struct{} + +func (engine *emptyStatsEngine) GetInstanceMetrics() (*ecstcs.MetricsMetadata, []*ecstcs.TaskMetric, error) { + return nil, nil, fmt.Errorf("empty stats") +} + type idleStatsEngine struct{} func (engine *idleStatsEngine) GetInstanceMetrics() (*ecstcs.MetricsMetadata, []*ecstcs.TaskMetric, error) { @@ -150,6 +157,14 @@ func TestPublishMetricsRequest(t *testing.T) { cs.Close() } +func TestPublishMetricsOnceEmptyStatsError(t *testing.T) { + cs := clientServer{ + statsEngine: &emptyStatsEngine{}, + } + err := cs.publishMetricsOnce() + + assert.Error(t, err, "Failed: expecting publishMerticOnce return err ") +} func TestPublishOnceIdleStatsEngine(t *testing.T) { cs := clientServer{ From d179457160dde947bc7bc838efb891e251303df2 Mon Sep 17 00:00:00 2001 From: Noah Meyerhans Date: Tue, 31 Jan 2017 13:05:55 -0800 Subject: [PATCH 03/27] Update tcshandler logging for seelog. --- agent/tcs/handler/handler.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/agent/tcs/handler/handler.go b/agent/tcs/handler/handler.go index 22ce097b2e6..286f5332d59 100644 --- a/agent/tcs/handler/handler.go +++ b/agent/tcs/handler/handler.go @@ -44,7 +44,7 @@ const ( func StartMetricsSession(params TelemetrySessionParams) { disabled, err := params.isTelemetryDisabled() if err != nil { - log.Warn("Error getting telemetry config", "err", err) + log.Warnf("Error getting telemetry config: %v", err) return } @@ -52,12 +52,12 @@ func StartMetricsSession(params TelemetrySessionParams) { statsEngine := stats.NewDockerStatsEngine(params.Cfg, params.DockerClient, params.ContainerChangeEventStream) err := statsEngine.MustInit(params.TaskEngine, params.Cfg.Cluster, params.ContainerInstanceArn) if err != nil { - log.Warn("Error initializing metrics engine", "err", err) + log.Warnf("Error initializing metrics engine: %v", err) return } err = StartSession(params, statsEngine) if err != nil { - log.Warn("Error starting metrics session with backend", "err", err) + log.Warnf("Error starting metrics session with backend: %v", err) return } } else { @@ -76,7 +76,7 @@ func StartSession(params TelemetrySessionParams, statsEngine stats.Engine) error if tcsError == nil || tcsError == io.EOF { backoff.Reset() } else { - log.Info("Error from tcs; backing off", "err", tcsError) + log.Infof("Error from tcs; backing off: %v", tcsError) params.time().Sleep(backoff.Duration()) } } @@ -85,10 +85,10 @@ func StartSession(params TelemetrySessionParams, statsEngine stats.Engine) error func startTelemetrySession(params TelemetrySessionParams, statsEngine stats.Engine) error { tcsEndpoint, err := params.ECSClient.DiscoverTelemetryEndpoint(params.ContainerInstanceArn) if err != nil { - log.Error("Unable to discover poll endpoint", "err", err) + log.Errorf("Unable to discover poll endpoint: ", err) return err } - log.Debug("Connecting to TCS endpoint " + tcsEndpoint) + log.Debugf("Connecting to TCS endpoint %v", tcsEndpoint) url := formatURL(tcsEndpoint, params.Cfg.Cluster, params.ContainerInstanceArn) return startSession(url, params.Cfg.AWSRegion, params.CredentialProvider, params.AcceptInvalidCert, statsEngine, defaultHeartbeatTimeout, defaultHeartbeatJitter, defaultPublishMetricsInterval, params.DeregisterInstanceEventStream) } @@ -117,7 +117,7 @@ func startSession(url string, region string, credentialProvider *credentials.Cre client.AddRequestHandler(ackPublishMetricHandler(timer)) err = client.Connect() if err != nil { - log.Error("Error connecting to TCS: " + err.Error()) + log.Errorf("Error connecting to TCS: %v", err.Error()) return err } return client.Serve() From 7219a65236be464a49c49692e12ceb9a80c73b6e Mon Sep 17 00:00:00 2001 From: Euan Kemp Date: Wed, 1 Feb 2017 08:28:37 -0800 Subject: [PATCH 04/27] travis: remove hacky symlinking Travis now provides a feature that fixes the issue the hack script was working around. --- .travis.yml | 8 ++++---- scripts/hack/symlink-gopath-travisci | 20 -------------------- 2 files changed, 4 insertions(+), 24 deletions(-) delete mode 100755 scripts/hack/symlink-gopath-travisci diff --git a/.travis.yml b/.travis.yml index 9594aed966f..de220c3868d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,9 @@ language: go sudo: false go: - - 1.7 -before_install: ./scripts/hack/symlink-gopath-travisci + - 1.7 +go_import_path: github.com/aws/amazon-ecs-agent + install: make get-deps script: - # Full tests require building/running docker containers; short only for now - - cd $HOME/gopath/src/github.com/aws/amazon-ecs-agent; make test + - make test diff --git a/scripts/hack/symlink-gopath-travisci b/scripts/hack/symlink-gopath-travisci deleted file mode 100755 index 5c1103d4b62..00000000000 --- a/scripts/hack/symlink-gopath-travisci +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -# Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You may -# not use this file except in compliance with the License. A copy of the -# License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is distributed -# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -# express or implied. See the License for the specific language governing -# permissions and limitations under the License. - -# This script is meant to make all package available in the 'canonical' location -# when running on travis-ci -if [[ ! -d "$HOME/gopath/src/github.com/aws/amazon-ecs-agent" ]]; then - mkdir -p "$HOME/gopath/src/github.com/aws" - ln -s "$(pwd)" "$HOME/gopath/src/github.com/aws/amazon-ecs-agent" -fi From 6a9584aca2d1f5584da2ccdc2aa300ef35a08ec3 Mon Sep 17 00:00:00 2001 From: Samuel Karp Date: Fri, 3 Feb 2017 14:21:43 -0800 Subject: [PATCH 05/27] engine: Increase start and create timeouts --- agent/engine/docker_container_engine.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/agent/engine/docker_container_engine.go b/agent/engine/docker_container_engine.go index 10548f5d9c7..adcc1136fa1 100644 --- a/agent/engine/docker_container_engine.go +++ b/agent/engine/docker_container_engine.go @@ -45,8 +45,8 @@ const ( // ListContainersTimeout is the timeout for the ListContainers API. ListContainersTimeout = 10 * time.Minute pullImageTimeout = 2 * time.Hour - createContainerTimeout = 3 * time.Minute - startContainerTimeout = 1*time.Minute + 30*time.Second + createContainerTimeout = 4 * time.Minute + startContainerTimeout = 3 * time.Minute stopContainerTimeout = 30 * time.Second removeContainerTimeout = 5 * time.Minute inspectContainerTimeout = 30 * time.Second From 89930da536d59345c989ea06a27b0712994ecf4c Mon Sep 17 00:00:00 2001 From: Anirudh Aithal Date: Thu, 9 Feb 2017 13:38:52 -0800 Subject: [PATCH 06/27] Fix test batch container happy path unit test (#703) * Force steady state check in TestBatchContainerHappyPath. This should hopefully remove the flakiness reported in #662 * Refactor task engine tests to use testify and to extract the common createDockerEvent into a function of its own --- agent/engine/docker_task_engine_test.go | 293 +++++++++--------------- 1 file changed, 108 insertions(+), 185 deletions(-) diff --git a/agent/engine/docker_task_engine_test.go b/agent/engine/docker_task_engine_test.go index 228551e0add..2f9693adba4 100644 --- a/agent/engine/docker_task_engine_test.go +++ b/agent/engine/docker_task_engine_test.go @@ -58,10 +58,15 @@ func mocks(t *testing.T, cfg *config.Config) (*gomock.Controller, *MockDockerCli return ctrl, client, mockTime, taskEngine, credentialsManager, imageManager } -func TestBatchContainerHappyPath(t *testing.T) { +func createDockerEvent(status api.ContainerStatus) DockerContainerChangeEvent { + meta := DockerContainerMetadata{ + DockerID: "containerId", + } + return DockerContainerChangeEvent{Status: status, DockerContainerMetadata: meta} +} +func TestBatchContainerHappyPath(t *testing.T) { ctrl, client, mockTime, taskEngine, credentialsManager, imageManager := mocks(t, &defaultConfig) - defer ctrl.Finish() roleCredentials := &credentials.TaskIAMRoleCredentials{ @@ -74,14 +79,9 @@ func TestBatchContainerHappyPath(t *testing.T) { sleepTask.SetCredentialsId(credentialsID) eventStream := make(chan DockerContainerChangeEvent) - eventsReported := sync.WaitGroup{} - - dockerEvent := func(status api.ContainerStatus) DockerContainerChangeEvent { - meta := DockerContainerMetadata{ - DockerID: "containerId", - } - return DockerContainerChangeEvent{Status: status, DockerContainerMetadata: meta} - } + // createStartEventsReported is used to force the test to wait until the container created and started + // events are processed + createStartEventsReported := sync.WaitGroup{} client.EXPECT().Version() client.EXPECT().ContainerEvents(gomock.Any()).Return(eventStream, nil) @@ -92,12 +92,12 @@ func TestBatchContainerHappyPath(t *testing.T) { imageManager.EXPECT().RecordContainerReference(container).Return(nil) imageManager.EXPECT().GetImageStateFromImageName(gomock.Any()).Return(nil) dockerConfig, err := sleepTask.DockerConfig(container) - // Container config should get updated with this during PostUnmarshalTask - credentialsEndpointEnvValue := roleCredentials.IAMRoleCredentials.GenerateCredentialsEndpointRelativeURI() - dockerConfig.Env = append(dockerConfig.Env, "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI="+credentialsEndpointEnvValue) if err != nil { t.Fatal(err) } + // Container config should get updated with this during PostUnmarshalTask + credentialsEndpointEnvValue := roleCredentials.IAMRoleCredentials.GenerateCredentialsEndpointRelativeURI() + dockerConfig.Env = append(dockerConfig.Env, "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI="+credentialsEndpointEnvValue) // Container config should get updated with this during CreateContainer dockerConfig.Labels["com.amazonaws.ecs.task-arn"] = sleepTask.Arn dockerConfig.Labels["com.amazonaws.ecs.container-name"] = container.Name @@ -113,43 +113,45 @@ func TestBatchContainerHappyPath(t *testing.T) { // sleep5 task contains only one container. Just assign // the containerName to createdContainerName createdContainerName = containerName - eventsReported.Add(1) + createStartEventsReported.Add(1) go func() { - eventStream <- dockerEvent(api.ContainerCreated) - eventsReported.Done() + eventStream <- createDockerEvent(api.ContainerCreated) + createStartEventsReported.Done() }() }).Return(DockerContainerMetadata{DockerID: "containerId"}) client.EXPECT().StartContainer("containerId", startContainerTimeout).Do( func(id string, timeout time.Duration) { - eventsReported.Add(1) + createStartEventsReported.Add(1) go func() { - eventStream <- dockerEvent(api.ContainerRunning) - eventsReported.Done() + eventStream <- createDockerEvent(api.ContainerRunning) + createStartEventsReported.Done() }() }).Return(DockerContainerMetadata{DockerID: "containerId"}) } + // steadyStateCheckWait is used to force the test to wait until the steady-state check + // has been invoked at least once + steadyStateCheckWait := sync.WaitGroup{} steadyStateVerify := make(chan time.Time, 1) cleanup := make(chan time.Time, 1) mockTime.EXPECT().Now().Do(func() time.Time { return time.Now() }).AnyTimes() - mockTime.EXPECT().After(steadyStateTaskVerifyInterval).Return(steadyStateVerify).AnyTimes() - mockTime.EXPECT().After(gomock.Any()).Return(cleanup).AnyTimes() + gomock.InOrder( + mockTime.EXPECT().After(steadyStateTaskVerifyInterval).Do(func(d time.Duration) { + steadyStateCheckWait.Done() + }).Return(steadyStateVerify), + mockTime.EXPECT().After(steadyStateTaskVerifyInterval).Return(steadyStateVerify).AnyTimes(), + ) err := taskEngine.Init() - taskEvents, contEvents := taskEngine.TaskEvents() - if err != nil { - t.Fatal(err) - } + assert.NoError(t, err) + taskEvents, contEvents := taskEngine.TaskEvents() + steadyStateCheckWait.Add(1) taskEngine.AddTask(sleepTask) - if (<-contEvents).Status != api.ContainerRunning { - t.Fatal("Expected container to run first") - } - if (<-taskEvents).Status != api.TaskRunning { - t.Fatal("And then task") - } + assert.Equal(t, (<-contEvents).Status, api.ContainerRunning, "Expected container to run first") + assert.Equal(t, (<-taskEvents).Status, api.TaskRunning, "Expected task to be RUNNING") select { case <-taskEvents: t.Fatal("Should be out of events") @@ -157,29 +159,38 @@ func TestBatchContainerHappyPath(t *testing.T) { t.Fatal("Should be out of events") default: } - eventsReported.Wait() + + // Wait for container create and start events to be processed + createStartEventsReported.Wait() + // Wait for steady state check to be invoked + steadyStateCheckWait.Wait() + mockTime.EXPECT().After(gomock.Any()).Return(cleanup).AnyTimes() + client.EXPECT().DescribeContainer(gomock.Any()).AnyTimes() + // Wait for all events to be consumed prior to moving it towards stopped; we // don't want to race the below with these or we'll end up with the "going // backwards in state" stop and we haven't 'expect'd for that exitCode := 0 // And then docker reports that sleep died, as sleep is wont to do - eventStream <- DockerContainerChangeEvent{Status: api.ContainerStopped, DockerContainerMetadata: DockerContainerMetadata{DockerID: "containerId", ExitCode: &exitCode}} + eventStream <- DockerContainerChangeEvent{ + Status: api.ContainerStopped, + DockerContainerMetadata: DockerContainerMetadata{ + DockerID: "containerId", + ExitCode: &exitCode, + }, + } steadyStateVerify <- time.Now() if cont := <-contEvents; cont.Status != api.ContainerStopped { t.Fatal("Expected container to stop first") - if *cont.ExitCode != 0 { - t.Fatal("Exit code should be present") - } - } - if (<-taskEvents).Status != api.TaskStopped { - t.Fatal("And then task") + assert.Equal(t, *cont.ExitCode, 0, "Exit code should be present") } + assert.Equal(t, (<-taskEvents).Status, api.TaskStopped, "Task is not in STOPPED state") // Extra events should not block forever; duplicate acs and docker events are possible - go func() { eventStream <- dockerEvent(api.ContainerStopped) }() - go func() { eventStream <- dockerEvent(api.ContainerStopped) }() + go func() { eventStream <- createDockerEvent(api.ContainerStopped) }() + go func() { eventStream <- createDockerEvent(api.ContainerStopped) }() sleepTaskStop := testdata.LoadTask("sleep5") sleepTaskStop.SetCredentialsId(credentialsID) @@ -190,18 +201,15 @@ func TestBatchContainerHappyPath(t *testing.T) { taskEngine.AddTask(sleepTaskStop) // Expect a bunch of steady state 'poll' describes when we trigger cleanup - client.EXPECT().DescribeContainer(gomock.Any()).AnyTimes() client.EXPECT().RemoveContainer(gomock.Any(), gomock.Any()).Do( func(removedContainerName string, timeout time.Duration) { - if createdContainerName != removedContainerName { - t.Errorf("Container name mismatch, created: %s, removed: %s", createdContainerName, removedContainerName) - } + assert.Equal(t, createdContainerName, removedContainerName, "Container name mismatch") }).Return(nil) imageManager.EXPECT().RemoveContainerReferenceFromImageState(gomock.Any()) // trigger cleanup cleanup <- time.Now() - go func() { eventStream <- dockerEvent(api.ContainerStopped) }() + go func() { eventStream <- createDockerEvent(api.ContainerStopped) }() // Wait for the task to actually be dead; if we just fallthrough immediately, // the remove might not have happened (expectation failure) @@ -223,13 +231,6 @@ func TestRemoveEvents(t *testing.T) { eventStream := make(chan DockerContainerChangeEvent) eventsReported := sync.WaitGroup{} - dockerEvent := func(status api.ContainerStatus) DockerContainerChangeEvent { - meta := DockerContainerMetadata{ - DockerID: "containerId", - } - return DockerContainerChangeEvent{Status: status, DockerContainerMetadata: meta} - } - client.EXPECT().Version() client.EXPECT().ContainerEvents(gomock.Any()).Return(eventStream, nil) var createdContainerName string @@ -245,7 +246,7 @@ func TestRemoveEvents(t *testing.T) { createdContainerName = containerName eventsReported.Add(1) go func() { - eventStream <- dockerEvent(api.ContainerCreated) + eventStream <- createDockerEvent(api.ContainerCreated) eventsReported.Done() }() }).Return(DockerContainerMetadata{DockerID: "containerId"}) @@ -254,7 +255,7 @@ func TestRemoveEvents(t *testing.T) { func(id string, timeout time.Duration) { eventsReported.Add(1) go func() { - eventStream <- dockerEvent(api.ContainerRunning) + eventStream <- createDockerEvent(api.ContainerRunning) eventsReported.Done() }() }).Return(DockerContainerMetadata{DockerID: "containerId"}) @@ -267,19 +268,12 @@ func TestRemoveEvents(t *testing.T) { testTime.EXPECT().After(gomock.Any()).Return(cleanup).AnyTimes() err := taskEngine.Init() + assert.NoError(t, err) taskEvents, contEvents := taskEngine.TaskEvents() - if err != nil { - t.Fatal(err) - } - taskEngine.AddTask(sleepTask) - if (<-contEvents).Status != api.ContainerRunning { - t.Fatal("Expected container to run first") - } - if (<-taskEvents).Status != api.TaskRunning { - t.Fatal("And then task") - } + assert.Equal(t, (<-contEvents).Status, api.ContainerRunning, "Expected container to run first") + assert.Equal(t, (<-taskEvents).Status, api.TaskRunning, "Expected task to be RUNNING") select { case <-taskEvents: t.Fatal("Should be out of events") @@ -305,13 +299,9 @@ func TestRemoveEvents(t *testing.T) { if cont := <-contEvents; cont.Status != api.ContainerStopped { t.Fatal("Expected container to stop first") - if *cont.ExitCode != 0 { - t.Fatal("Exit code should be present") - } - } - if (<-taskEvents).Status != api.TaskStopped { - t.Fatal("And then task") + assert.Equal(t, *cont.ExitCode, 0, "Exit code should be present") } + assert.Equal(t, (<-taskEvents).Status, api.TaskStopped, "Expected task to be STOPPED") sleepTaskStop := testdata.LoadTask("sleep5") sleepTaskStop.SetDesiredStatus(api.TaskStopped) @@ -320,12 +310,10 @@ func TestRemoveEvents(t *testing.T) { client.EXPECT().DescribeContainer(gomock.Any()).AnyTimes() client.EXPECT().RemoveContainer(gomock.Any(), gomock.Any()).Do( func(removedContainerName string, timeout time.Duration) { - if createdContainerName != removedContainerName { - t.Errorf("Container name mismatch, created: %s, removed: %s", createdContainerName, removedContainerName) - } + assert.Equal(t, createdContainerName, removedContainerName, "Container name mismatch") // Emit a couple events for the task before the remove finishes; make sure this gets handled appropriately - eventStream <- dockerEvent(api.ContainerStopped) - eventStream <- dockerEvent(api.ContainerStopped) + eventStream <- createDockerEvent(api.ContainerStopped) + eventStream <- createDockerEvent(api.ContainerStopped) }).Return(nil) taskEngine.AddTask(sleepTaskStop) @@ -352,13 +340,6 @@ func TestStartTimeoutThenStart(t *testing.T) { eventStream := make(chan DockerContainerChangeEvent) testTime.EXPECT().After(gomock.Any()) - dockerEvent := func(status api.ContainerStatus) DockerContainerChangeEvent { - meta := DockerContainerMetadata{ - DockerID: "containerId", - } - return DockerContainerChangeEvent{Status: status, DockerContainerMetadata: meta} - } - client.EXPECT().Version() client.EXPECT().ContainerEvents(gomock.Any()).Return(eventStream, nil) for _, container := range sleepTask.Containers { @@ -381,7 +362,7 @@ func TestStartTimeoutThenStart(t *testing.T) { client.EXPECT().CreateContainer(dockerConfig, gomock.Any(), gomock.Any(), gomock.Any()).Do( func(x, y, z, timeout interface{}) { - go func() { eventStream <- dockerEvent(api.ContainerCreated) }() + go func() { eventStream <- createDockerEvent(api.ContainerCreated) }() }).Return(DockerContainerMetadata{DockerID: "containerId"}) client.EXPECT().StartContainer("containerId", startContainerTimeout).Return(DockerContainerMetadata{ @@ -390,23 +371,17 @@ func TestStartTimeoutThenStart(t *testing.T) { } err := taskEngine.Init() - taskEvents, contEvents := taskEngine.TaskEvents() - if err != nil { - t.Fatal(err) - } + assert.NoError(t, err) + taskEvents, contEvents := taskEngine.TaskEvents() taskEngine.AddTask(sleepTask) // Expect it to go to stopped contEvent := <-contEvents - if contEvent.Status != api.ContainerStopped { - t.Fatal("Expected container to timeout on start and stop") - } + assert.Equal(t, contEvent.Status, api.ContainerStopped, "Expected container to timeout on start and stop") taskEvent := <-taskEvents - if taskEvent.Status != api.TaskStopped { - t.Fatal("And then task") - } + assert.Equal(t, taskEvent.Status, api.TaskStopped, "Expect task to be STOPPED") select { case <-taskEvents: t.Fatal("Should be out of events") @@ -420,11 +395,11 @@ func TestStartTimeoutThenStart(t *testing.T) { Error: CannotStartContainerError{fmt.Errorf("cannot start container")}, }).AnyTimes() // Now surprise surprise, it actually did start! - eventStream <- dockerEvent(api.ContainerRunning) + eventStream <- createDockerEvent(api.ContainerRunning) // However, if it starts again, we should not see it be killed; no additional expect - eventStream <- dockerEvent(api.ContainerRunning) - eventStream <- dockerEvent(api.ContainerRunning) + eventStream <- createDockerEvent(api.ContainerRunning) + eventStream <- createDockerEvent(api.ContainerRunning) select { case <-taskEvents: @@ -444,13 +419,6 @@ func TestSteadyStatePoll(t *testing.T) { eventStream := make(chan DockerContainerChangeEvent) - dockerEvent := func(status api.ContainerStatus) DockerContainerChangeEvent { - meta := DockerContainerMetadata{ - DockerID: "containerId", - } - return DockerContainerChangeEvent{Status: status, DockerContainerMetadata: meta} - } - client.EXPECT().Version() client.EXPECT().ContainerEvents(gomock.Any()).Return(eventStream, nil) // set up expectations for each container in the task calling create + start @@ -473,7 +441,7 @@ func TestSteadyStatePoll(t *testing.T) { func(x, y, z, timeout interface{}) { go func() { wait.Add(1) - eventStream <- dockerEvent(api.ContainerCreated) + eventStream <- createDockerEvent(api.ContainerCreated) wait.Done() }() }).Return(DockerContainerMetadata{DockerID: "containerId"}) @@ -482,7 +450,7 @@ func TestSteadyStatePoll(t *testing.T) { func(id string, timeout time.Duration) { go func() { wait.Add(1) - eventStream <- dockerEvent(api.ContainerRunning) + eventStream <- createDockerEvent(api.ContainerRunning) wait.Done() }() }).Return(DockerContainerMetadata{DockerID: "containerId"}) @@ -492,8 +460,8 @@ func TestSteadyStatePoll(t *testing.T) { testTime.EXPECT().After(steadyStateTaskVerifyInterval).Return(steadyStateVerify).AnyTimes() testTime.EXPECT().After(gomock.Any()).AnyTimes() err := taskEngine.Init() // start the task engine - taskEvents, contEvents := taskEngine.TaskEvents() assert.Nil(t, err) + taskEvents, contEvents := taskEngine.TaskEvents() taskEngine.AddTask(sleepTask) // actually add the task we created @@ -567,9 +535,7 @@ func TestStopWithPendingStops(t *testing.T) { client.EXPECT().Version() client.EXPECT().ContainerEvents(gomock.Any()).Return(eventStream, nil) err := taskEngine.Init() - if err != nil { - t.Fatal(err) - } + assert.NoError(t, err) taskEvents, contEvents := taskEngine.TaskEvents() go func() { for { @@ -622,13 +588,10 @@ func TestCreateContainerForceSave(t *testing.T) { gomock.InOrder( saver.EXPECT().ForceSave().Do(func() interface{} { task, ok := taskEngine.state.TaskByArn(sleepTask.Arn) - if task == nil || !ok { - t.Fatalf("Expected task with arn %s", sleepTask.Arn) - } + assert.True(t, ok, "Expected task with ARN: ", sleepTask.Arn) + assert.NotNil(t, task, "Expected task with ARN: ", sleepTask.Arn) _, ok = task.ContainerByName("sleep5") - if !ok { - t.Error("Expected container sleep5") - } + assert.True(t, ok, "Expected container sleep5") return nil }), client.EXPECT().CreateContainer(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()), @@ -684,13 +647,6 @@ func TestTaskTransitionWhenStopContainerTimesout(t *testing.T) { eventStream := make(chan DockerContainerChangeEvent) - dockerEvent := func(status api.ContainerStatus) DockerContainerChangeEvent { - meta := DockerContainerMetadata{ - DockerID: "containerId", - } - return DockerContainerChangeEvent{Status: status, DockerContainerMetadata: meta} - } - client.EXPECT().Version() client.EXPECT().ContainerEvents(gomock.Any()).Return(eventStream, nil) mockTime.EXPECT().After(gomock.Any()).AnyTimes() @@ -719,14 +675,14 @@ func TestTaskTransitionWhenStopContainerTimesout(t *testing.T) { client.EXPECT().CreateContainer(dockerConfig, gomock.Any(), gomock.Any(), gomock.Any()).Do( func(x, y, z, timeout interface{}) { - go func() { eventStream <- dockerEvent(api.ContainerCreated) }() + go func() { eventStream <- createDockerEvent(api.ContainerCreated) }() }).Return(DockerContainerMetadata{DockerID: "containerId"}) gomock.InOrder( client.EXPECT().StartContainer("containerId", startContainerTimeout).Do( func(id string, timeout time.Duration) { go func() { - eventStream <- dockerEvent(api.ContainerRunning) + eventStream <- createDockerEvent(api.ContainerRunning) }() }).Return(DockerContainerMetadata{DockerID: "containerId"}), @@ -743,29 +699,23 @@ func TestTaskTransitionWhenStopContainerTimesout(t *testing.T) { // Emit 'ContainerStopped' event to the container event stream // This should cause the container and the task to transition // to 'STOPPED' - eventStream <- dockerEvent(api.ContainerStopped) + eventStream <- createDockerEvent(api.ContainerStopped) }() }).Return(containerStopTimeoutError).AnyTimes(), ) } err := taskEngine.Init() + assert.NoError(t, err) taskEvents, contEvents := taskEngine.TaskEvents() - if err != nil { - t.Fatalf("Error getting event streams from engine: %v", err) - } go taskEngine.AddTask(sleepTask) // wait for task running contEvent := <-contEvents - if contEvent.Status != api.ContainerRunning { - t.Errorf("Expected container to be running, got: %v", contEvent) - } + assert.Equal(t, contEvent.Status, api.ContainerRunning, "Expected container to be running") taskEvent := <-taskEvents - if taskEvent.Status != api.TaskRunning { - t.Errorf("Expected task to be running, got: %v", taskEvent) - } + assert.Equal(t, taskEvent.Status, api.TaskRunning, "Expected task to be running") // Set the task desired status to be stopped and StopContainer will be called updateSleepTask := *sleepTask @@ -791,13 +741,9 @@ func TestTaskTransitionWhenStopContainerTimesout(t *testing.T) { // StopContainer was called again and received stop event from docker event stream // Expect it to go to stopped contEvent = <-contEvents - if contEvent.Status != api.ContainerStopped { - t.Errorf("Expected container to timeout on start and stop, got: %v", contEvent) - } + assert.Equal(t, contEvent.Status, api.ContainerStopped, "Expected container to timeout on start and stop") taskEvent = <-taskEvents - if taskEvent.Status != api.TaskStopped { - t.Errorf("Expected task to be stopped, got: %v", taskEvent) - } + assert.Equal(t, taskEvent.Status, api.TaskStopped, "Expected task to be stopped") select { case <-taskEvents: @@ -818,13 +764,6 @@ func TestTaskTransitionWhenStopContainerReturnsUnretriableError(t *testing.T) { sleepTask := testdata.LoadTask("sleep5") eventStream := make(chan DockerContainerChangeEvent) - dockerEvent := func(status api.ContainerStatus) DockerContainerChangeEvent { - meta := DockerContainerMetadata{ - DockerID: "containerId", - } - return DockerContainerChangeEvent{Status: status, DockerContainerMetadata: meta} - } - client.EXPECT().Version() client.EXPECT().ContainerEvents(gomock.Any()).Return(eventStream, nil) mockTime.EXPECT().After(gomock.Any()).AnyTimes() @@ -840,7 +779,7 @@ func TestTaskTransitionWhenStopContainerReturnsUnretriableError(t *testing.T) { func(x, y, z, timeout interface{}) { eventsReported.Add(1) go func() { - eventStream <- dockerEvent(api.ContainerCreated) + eventStream <- createDockerEvent(api.ContainerCreated) eventsReported.Done() }() }).Return(DockerContainerMetadata{DockerID: "containerId"}), @@ -849,7 +788,7 @@ func TestTaskTransitionWhenStopContainerReturnsUnretriableError(t *testing.T) { func(id string, timeout time.Duration) { eventsReported.Add(1) go func() { - eventStream <- dockerEvent(api.ContainerRunning) + eventStream <- createDockerEvent(api.ContainerRunning) eventsReported.Done() }() }).Return(DockerContainerMetadata{DockerID: "containerId"}), @@ -1042,9 +981,9 @@ func TestCapabilitiesECR(t *testing.T) { capMap[capability] = true } - if _, ok := capMap["com.amazonaws.ecs.capability.ecr-auth"]; !ok { - t.Errorf("Could not find ECR capability when expected; got capabilities %v", capabilities) - } + _, ok := capMap["com.amazonaws.ecs.capability.ecr-auth"] + assert.True(t, ok, "Could not find ECR capability when expected; got capabilities %v", capabilities) + } func TestCapabilitiesTaskIAMRoleForSupportedDockerVersion(t *testing.T) { @@ -1064,9 +1003,8 @@ func TestCapabilitiesTaskIAMRoleForSupportedDockerVersion(t *testing.T) { capMap[capability] = true } - if _, ok := capMap["com.amazonaws.ecs.capability.task-iam-role"]; !ok { - t.Errorf("Could not find iam capability when expected; got capabilities %v", capabilities) - } + ok := capMap["com.amazonaws.ecs.capability.task-iam-role"] + assert.True(t, ok, "Could not find iam capability when expected; got capabilities %v", capabilities) } func TestCapabilitiesTaskIAMRoleForUnSupportedDockerVersion(t *testing.T) { @@ -1086,9 +1024,8 @@ func TestCapabilitiesTaskIAMRoleForUnSupportedDockerVersion(t *testing.T) { capMap[capability] = true } - if _, ok := capMap["com.amazonaws.ecs.capability.task-iam-role"]; ok { - t.Errorf("task-iam-role capability set for unsupported docker version") - } + _, ok := capMap["com.amazonaws.ecs.capability.task-iam-role"] + assert.False(t, ok, "task-iam-role capability set for unsupported docker version") } func TestCapabilitiesTaskIAMRoleNetworkHostForSupportedDockerVersion(t *testing.T) { @@ -1108,9 +1045,8 @@ func TestCapabilitiesTaskIAMRoleNetworkHostForSupportedDockerVersion(t *testing. capMap[capability] = true } - if _, ok := capMap["com.amazonaws.ecs.capability.task-iam-role-network-host"]; !ok { - t.Errorf("Could not find iam capability when expected; got capabilities %v", capabilities) - } + _, ok := capMap["com.amazonaws.ecs.capability.task-iam-role-network-host"] + assert.True(t, ok, "Could not find iam capability when expected; got capabilities %v", capabilities) } func TestCapabilitiesTaskIAMRoleNetworkHostForUnSupportedDockerVersion(t *testing.T) { @@ -1130,9 +1066,8 @@ func TestCapabilitiesTaskIAMRoleNetworkHostForUnSupportedDockerVersion(t *testin capMap[capability] = true } - if _, ok := capMap["com.amazonaws.ecs.capability.task-iam-role-network-host"]; ok { - t.Errorf("task-iam-role capability set for unsupported docker version") - } + _, ok := capMap["com.amazonaws.ecs.capability.task-iam-role-network-host"] + assert.False(t, ok, "task-iam-role capability set for unsupported docker version") } func TestGetTaskByArn(t *testing.T) { @@ -1150,9 +1085,7 @@ func TestGetTaskByArn(t *testing.T) { imageManager.EXPECT().GetImageStateFromImageName(gomock.Any()).AnyTimes() client.EXPECT().PullImage(gomock.Any(), gomock.Any()).AnyTimes() // TODO change to MaxTimes(1) err := taskEngine.Init() - if err != nil { - t.Fatal(err) - } + assert.Nil(t, err) defer taskEngine.Disable() sleepTask := testdata.LoadTask("sleep5") @@ -1160,14 +1093,10 @@ func TestGetTaskByArn(t *testing.T) { taskEngine.AddTask(sleepTask) _, found := taskEngine.GetTaskByArn(sleepTaskArn) - if !found { - t.Fatalf("Task %s not found", sleepTaskArn) - } + assert.True(t, found, "Task %s not found", sleepTaskArn) _, found = taskEngine.GetTaskByArn(sleepTaskArn + "arn") - if found { - t.Fatal("Task with invalid arn found in the task engine") - } + assert.False(t, found, "Task with invalid arn found in the task engine") } func TestEngineEnableConcurrentPull(t *testing.T) { @@ -1177,16 +1106,11 @@ func TestEngineEnableConcurrentPull(t *testing.T) { client.EXPECT().Version().Return("1.11.1", nil) client.EXPECT().ContainerEvents(gomock.Any()) err := taskEngine.Init() - if err != nil { - t.Fatal(err) - - } + assert.Nil(t, err) dockerTaskEngine, _ := taskEngine.(*DockerTaskEngine) - - if !dockerTaskEngine.enableConcurrentPull { - t.Error("Task engine should be able to perform concurrent pulling for docker version >= 1.11.1") - } + assert.True(t, dockerTaskEngine.enableConcurrentPull, + "Task engine should be able to perform concurrent pulling for docker version >= 1.11.1") } func TestEngineDisableConcurrentPull(t *testing.T) { @@ -1202,7 +1126,6 @@ func TestEngineDisableConcurrentPull(t *testing.T) { } dockerTaskEngine, _ := taskEngine.(*DockerTaskEngine) - if dockerTaskEngine.enableConcurrentPull { - t.Error("Task engine should not be able to perform concurrent pulling for version < 1.11.1") - } + assert.False(t, dockerTaskEngine.enableConcurrentPull, + "Task engine should not be able to perform concurrent pulling for version < 1.11.1") } From e66feedd2f228b7e586b335626c5e166ab9f443c Mon Sep 17 00:00:00 2001 From: richardpen Date: Tue, 24 Jan 2017 23:48:57 +0000 Subject: [PATCH 07/27] Fixed a race condition that can cause agent panici --- agent/engine/image/types.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/agent/engine/image/types.go b/agent/engine/image/types.go index f7e038302a0..644d05c1630 100644 --- a/agent/engine/image/types.go +++ b/agent/engine/image/types.go @@ -14,6 +14,7 @@ package image import ( + "encoding/json" "fmt" "sync" "time" @@ -98,3 +99,18 @@ func (imageState *ImageState) RemoveContainerReference(container *api.Container) } return fmt.Errorf("Container reference is not found in the image state container: %s", container.String()) } + +func (imageState *ImageState) MarshalJSON() ([]byte, error) { + imageState.updateLock.Lock() + defer imageState.updateLock.Unlock() + + return json.Marshal(&struct { + Image *Image + PulledAt time.Time + LastUsedAt time.Time + }{ + Image: imageState.Image, + PulledAt: imageState.PulledAt, + LastUsedAt: imageState.LastUsedAt, + }) +} From d43880f525a17730b7c7bcb2d21ff5eb7816315f Mon Sep 17 00:00:00 2001 From: richardpen Date: Wed, 25 Jan 2017 22:48:47 +0000 Subject: [PATCH 08/27] Increase the task life duration to fix CleanupDoesNotDeadlock functional test --- .../task-definition.json | 20 +++++++++---------- .../tests/functionaltests_windows_test.go | 12 ++++++++--- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/agent/functional_tests/testdata/taskdefinitions/ten-containers-windows/task-definition.json b/agent/functional_tests/testdata/taskdefinitions/ten-containers-windows/task-definition.json index 8c5376b5aba..8a8797625be 100644 --- a/agent/functional_tests/testdata/taskdefinitions/ten-containers-windows/task-definition.json +++ b/agent/functional_tests/testdata/taskdefinitions/ten-containers-windows/task-definition.json @@ -6,69 +6,69 @@ "memory": 10, "cpu": 0, "entryPoint": ["powershell"], - "command": ["sleep", "20"] + "command": ["sleep", "300"] }, { "name": "2", "image": "microsoft/windowsservercore:latest", "memory": 10, "cpu": 0, "entryPoint": ["powershell"], - "command": ["sleep", "20"] + "command": ["sleep", "300"] }, { "name": "3", "image": "microsoft/windowsservercore:latest", "memory": 10, "cpu": 0, "entryPoint": ["powershell"], - "command": ["sleep", "20"] + "command": ["sleep", "300"] }, { "name": "4", "image": "microsoft/windowsservercore:latest", "memory": 10, "cpu": 0, "entryPoint": ["powershell"], - "command": ["sleep", "20"] + "command": ["sleep", "300"] }, { "name": "5", "image": "microsoft/windowsservercore:latest", "memory": 10, "cpu": 0, "entryPoint": ["powershell"], - "command": ["sleep", "20"] + "command": ["sleep", "300"] }, { "name": "6", "image": "microsoft/windowsservercore:latest", "memory": 10, "cpu": 0, "entryPoint": ["powershell"], - "command": ["sleep", "20"] + "command": ["sleep", "300"] }, { "name": "7", "image": "microsoft/windowsservercore:latest", "memory": 10, "cpu": 0, "entryPoint": ["powershell"], - "command": ["sleep", "20"] + "command": ["sleep", "300"] }, { "name": "8", "image": "microsoft/windowsservercore:latest", "memory": 10, "cpu": 0, "entryPoint": ["powershell"], - "command": ["sleep", "20"] + "command": ["sleep", "300"] }, { "name": "9", "image": "microsoft/windowsservercore:latest", "memory": 10, "cpu": 0, "entryPoint": ["powershell"], - "command": ["sleep", "20"] + "command": ["sleep", "300"] }, { "name": "10", "image": "microsoft/windowsservercore:latest", "memory": 10, "cpu": 0, "entryPoint": ["powershell"], - "command": ["sleep", "20"] + "command": ["sleep", "300"] }] } diff --git a/agent/functional_tests/tests/functionaltests_windows_test.go b/agent/functional_tests/tests/functionaltests_windows_test.go index 82b004f4a91..3bf95c347ad 100644 --- a/agent/functional_tests/tests/functionaltests_windows_test.go +++ b/agent/functional_tests/tests/functionaltests_windows_test.go @@ -62,9 +62,10 @@ func TestTaskCleanupDoesNotDeadlock(t *testing.T) { t.Fatalf("Cycle %d: There was an error starting the Task: %v", i, err) } - // Added 1 minute delay to allow the task to be in running state - Required only on Windows - testTask.WaitRunning(1 * time.Minute) + // Wait for the task to be running + testTask.WaitRunning(2 * time.Minute) + // Make sure the task is running on the instance isTaskRunning, err := agent.WaitRunningViaIntrospection(testTask) if err != nil || !isTaskRunning { t.Fatalf("Cycle %d: Task should be RUNNING but is not: %v", i, err) @@ -76,7 +77,12 @@ func TestTaskCleanupDoesNotDeadlock(t *testing.T) { t.Fatalf("Cycle %d: Error resolving docker id for container in task: %v", i, err) } - // 2 minutes should be enough for the Task to have completed. If the task has not + err = testTask.Stop() + if err != nil { + t.Fatalf("Cycle %d: Failed to stop task: %v", i, err) + } + + // 2 minutes should be enough for the Task to be stopped. If the task has not // completed and is in PENDING, the agent is most likely deadlocked. err = testTask.WaitStopped(2 * time.Minute) if err != nil { From de17a2c18341853d13afef50354fd806985806c3 Mon Sep 17 00:00:00 2001 From: Petersen Date: Wed, 14 Dec 2016 15:53:46 -0800 Subject: [PATCH 09/27] updating gorilla/websocket --- agent/Godeps/Godeps.json | 3 +- .../github.com/gorilla/websocket/.gitignore | 3 + .../github.com/gorilla/websocket/.travis.yml | 20 +- .../github.com/gorilla/websocket/README.md | 15 +- .../github.com/gorilla/websocket/client.go | 337 +++++++-- .../gorilla/websocket/compression.go | 85 +++ .../github.com/gorilla/websocket/conn.go | 710 ++++++++++++------ .../github.com/gorilla/websocket/conn_read.go | 18 + .../gorilla/websocket/conn_read_legacy.go | 21 + .../github.com/gorilla/websocket/doc.go | 78 +- .../github.com/gorilla/websocket/json.go | 8 +- .../github.com/gorilla/websocket/mask.go | 61 ++ .../github.com/gorilla/websocket/server.go | 66 +- .../github.com/gorilla/websocket/util.go | 196 ++++- 14 files changed, 1237 insertions(+), 384 deletions(-) create mode 100644 agent/vendor/github.com/gorilla/websocket/compression.go create mode 100644 agent/vendor/github.com/gorilla/websocket/conn_read.go create mode 100644 agent/vendor/github.com/gorilla/websocket/conn_read_legacy.go create mode 100644 agent/vendor/github.com/gorilla/websocket/mask.go diff --git a/agent/Godeps/Godeps.json b/agent/Godeps/Godeps.json index 2b92d6239d8..3c9fc2dcb1b 100644 --- a/agent/Godeps/Godeps.json +++ b/agent/Godeps/Godeps.json @@ -261,7 +261,8 @@ }, { "ImportPath": "github.com/gorilla/websocket", - "Rev": "87f6f6a22ebfbc3f89b9ccdc7fddd1b914c095f9" + "Comment": "v1.0.0-40-g0868951", + "Rev": "0868951cdb8e69bc42df4598bdc6164ff2f1a072" }, { "ImportPath": "github.com/hashicorp/go-cleanhttp", diff --git a/agent/vendor/github.com/gorilla/websocket/.gitignore b/agent/vendor/github.com/gorilla/websocket/.gitignore index 00268614f04..ac710204fa1 100644 --- a/agent/vendor/github.com/gorilla/websocket/.gitignore +++ b/agent/vendor/github.com/gorilla/websocket/.gitignore @@ -20,3 +20,6 @@ _cgo_export.* _testmain.go *.exe + +.idea/ +*.iml \ No newline at end of file diff --git a/agent/vendor/github.com/gorilla/websocket/.travis.yml b/agent/vendor/github.com/gorilla/websocket/.travis.yml index 8687342e9d4..4ea1e7a1fc1 100644 --- a/agent/vendor/github.com/gorilla/websocket/.travis.yml +++ b/agent/vendor/github.com/gorilla/websocket/.travis.yml @@ -1,6 +1,18 @@ language: go +sudo: false -go: - - 1.1 - - 1.2 - - tip +matrix: + include: + - go: 1.4 + - go: 1.5 + - go: 1.6 + - go: 1.7 + - go: tip + allow_failures: + - go: tip + +script: + - go get -t -v ./... + - diff -u <(echo -n) <(gofmt -d .) + - go vet $(go list ./... | grep -v /vendor/) + - go test -v -race ./... diff --git a/agent/vendor/github.com/gorilla/websocket/README.md b/agent/vendor/github.com/gorilla/websocket/README.md index 9ca3ca8eb7f..33c3d2be3e3 100644 --- a/agent/vendor/github.com/gorilla/websocket/README.md +++ b/agent/vendor/github.com/gorilla/websocket/README.md @@ -3,10 +3,15 @@ Gorilla WebSocket is a [Go](http://golang.org/) implementation of the [WebSocket](http://www.rfc-editor.org/rfc/rfc6455.txt) protocol. +[![Build Status](https://travis-ci.org/gorilla/websocket.svg?branch=master)](https://travis-ci.org/gorilla/websocket) +[![GoDoc](https://godoc.org/github.com/gorilla/websocket?status.svg)](https://godoc.org/github.com/gorilla/websocket) + ### Documentation * [API Reference](http://godoc.org/github.com/gorilla/websocket) * [Chat example](https://github.com/gorilla/websocket/tree/master/examples/chat) +* [Command example](https://github.com/gorilla/websocket/tree/master/examples/command) +* [Client and server example](https://github.com/gorilla/websocket/tree/master/examples/echo) * [File watch example](https://github.com/gorilla/websocket/tree/master/examples/filewatch) ### Status @@ -30,8 +35,8 @@ subdirectory](https://github.com/gorilla/websocket/tree/master/examples/autobahn - - + + @@ -41,7 +46,7 @@ subdirectory](https://github.com/gorilla/websocket/tree/master/examples/autobahn - +
gorillago.netgithub.com/gorillagolang.org/x/net
RFC 6455 Features
Send pings and receive pongsYesNo
Get the type of a received data messageYesYes, see note 2
Other Features
Limit size of received messageYesNo
Compression ExtensionsExperimentalNo
Read message using io.ReaderYesNo, see note 3
Write message using io.WriteCloserYesNo, see note 3
@@ -50,10 +55,10 @@ Notes: 1. Large messages are fragmented in [Chrome's new WebSocket implementation](http://www.ietf.org/mail-archive/web/hybi/current/msg10503.html). 2. The application can get the type of a received data message by implementing - a [Codec marshal](http://godoc.org/code.google.com/p/go.net/websocket#Codec.Marshal) + a [Codec marshal](http://godoc.org/golang.org/x/net/websocket#Codec.Marshal) function. 3. The go.net io.Reader and io.Writer operate across WebSocket frame boundaries. Read returns when the input buffer is full or a frame boundary is - encountered, Each call to Write sends a single frame message. The Gorilla + encountered. Each call to Write sends a single frame message. The Gorilla io.Reader and io.WriteCloser operate on a single WebSocket message. diff --git a/agent/vendor/github.com/gorilla/websocket/client.go b/agent/vendor/github.com/gorilla/websocket/client.go index 3b5cac4612c..78d932877b5 100644 --- a/agent/vendor/github.com/gorilla/websocket/client.go +++ b/agent/vendor/github.com/gorilla/websocket/client.go @@ -5,8 +5,13 @@ package websocket import ( + "bufio" + "bytes" "crypto/tls" + "encoding/base64" "errors" + "io" + "io/ioutil" "net" "net/http" "net/url" @@ -18,6 +23,8 @@ import ( // invalid. var ErrBadHandshake = errors.New("websocket: bad handshake") +var errInvalidCompression = errors.New("websocket: invalid compression negotiation") + // NewClient creates a new client connection using the given net connection. // The URL u specifies the host and request URI. Use requestHeader to specify // the origin (Origin), subprotocols (Sec-WebSocket-Protocol) and cookies @@ -27,50 +34,17 @@ var ErrBadHandshake = errors.New("websocket: bad handshake") // If the WebSocket handshake fails, ErrBadHandshake is returned along with a // non-nil *http.Response so that callers can handle redirects, authentication, // etc. +// +// Deprecated: Use Dialer instead. func NewClient(netConn net.Conn, u *url.URL, requestHeader http.Header, readBufSize, writeBufSize int) (c *Conn, response *http.Response, err error) { - challengeKey, err := generateChallengeKey() - if err != nil { - return nil, nil, err - } - acceptKey := computeAcceptKey(challengeKey) - - c = newConn(netConn, false, readBufSize, writeBufSize) - p := c.writeBuf[:0] - p = append(p, "GET "...) - p = append(p, u.RequestURI()...) - p = append(p, " HTTP/1.1\r\nHost: "...) - p = append(p, u.Host...) - // "Upgrade" is capitalized for servers that do not use case insensitive - // comparisons on header tokens. - p = append(p, "\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Version: 13\r\nSec-WebSocket-Key: "...) - p = append(p, challengeKey...) - p = append(p, "\r\n"...) - for k, vs := range requestHeader { - for _, v := range vs { - p = append(p, k...) - p = append(p, ": "...) - p = append(p, v...) - p = append(p, "\r\n"...) - } - } - p = append(p, "\r\n"...) - - if _, err := netConn.Write(p); err != nil { - return nil, nil, err - } - - resp, err := http.ReadResponse(c.br, &http.Request{Method: "GET", URL: u}) - if err != nil { - return nil, nil, err - } - if resp.StatusCode != 101 || - !strings.EqualFold(resp.Header.Get("Upgrade"), "websocket") || - !strings.EqualFold(resp.Header.Get("Connection"), "upgrade") || - resp.Header.Get("Sec-Websocket-Accept") != acceptKey { - return nil, resp, ErrBadHandshake + d := Dialer{ + ReadBufferSize: readBufSize, + WriteBufferSize: writeBufSize, + NetDial: func(net, addr string) (net.Conn, error) { + return netConn, nil + }, } - c.subprotocol = resp.Header.Get("Sec-Websocket-Protocol") - return c, resp, nil + return d.Dial(u.String(), requestHeader) } // A Dialer contains options for connecting to WebSocket server. @@ -79,6 +53,12 @@ type Dialer struct { // NetDial is nil, net.Dial is used. NetDial func(network, addr string) (net.Conn, error) + // Proxy specifies a function to return a proxy for a given + // Request. If the function returns a non-nil error, the + // request is aborted with the provided error. + // If Proxy is nil or returns a nil *URL, no proxy is used. + Proxy func(*http.Request) (*url.URL, error) + // TLSClientConfig specifies the TLS configuration to use with tls.Client. // If nil, the default configuration is used. TLSClientConfig *tls.Config @@ -92,22 +72,30 @@ type Dialer struct { // Subprotocols specifies the client's requested subprotocols. Subprotocols []string + + // EnableCompression specifies if the client should attempt to negotiate + // per message compression (RFC 7692). Setting this value to true does not + // guarantee that compression will be supported. Currently only "no context + // takeover" modes are supported. + EnableCompression bool + + // Jar specifies the cookie jar. + // If Jar is nil, cookies are not sent in requests and ignored + // in responses. + Jar http.CookieJar } var errMalformedURL = errors.New("malformed ws or wss URL") -// parseURL parses the URL. The url.Parse function is not used here because -// url.Parse mangles the path. +// parseURL parses the URL. +// +// This function is a replacement for the standard library url.Parse function. +// In Go 1.4 and earlier, url.Parse loses information from the path. func parseURL(s string) (*url.URL, error) { // From the RFC: // // ws-URI = "ws:" "//" host [ ":" port ] path [ "?" query ] // wss-URI = "wss:" "//" host [ ":" port ] path [ "?" query ] - // - // We don't use the net/url parser here because the dialer interface does - // not provide a way for applications to work around percent deocding in - // the net/url parser. - var u url.URL switch { case strings.HasPrefix(s, "ws://"): @@ -120,11 +108,24 @@ func parseURL(s string) (*url.URL, error) { return nil, errMalformedURL } - u.Host = s - u.Opaque = "/" + if i := strings.Index(s, "?"); i >= 0 { + u.RawQuery = s[i+1:] + s = s[:i] + } + if i := strings.Index(s, "/"); i >= 0 { - u.Host = s[:i] u.Opaque = s[i:] + s = s[:i] + } else { + u.Opaque = "/" + } + + u.Host = s + + if strings.Contains(u.Host, "@") { + // Don't bother parsing user information because user information is + // not allowed in websocket URIs. + return nil, errMalformedURL } return &u, nil @@ -136,9 +137,12 @@ func hostPortNoPort(u *url.URL) (hostPort, hostNoPort string) { if i := strings.LastIndex(u.Host, ":"); i > strings.LastIndex(u.Host, "]") { hostNoPort = hostNoPort[:i] } else { - if u.Scheme == "wss" { + switch u.Scheme { + case "wss": + hostPort += ":443" + case "https": hostPort += ":443" - } else { + default: hostPort += ":80" } } @@ -146,7 +150,9 @@ func hostPortNoPort(u *url.URL) (hostPort, hostNoPort string) { } // DefaultDialer is a dialer with all fields set to the default zero values. -var DefaultDialer *Dialer +var DefaultDialer = &Dialer{ + Proxy: http.ProxyFromEnvironment, +} // Dial creates a new client connection. Use requestHeader to specify the // origin (Origin), subprotocols (Sec-WebSocket-Protocol) and cookies (Cookie). @@ -155,17 +161,106 @@ var DefaultDialer *Dialer // // If the WebSocket handshake fails, ErrBadHandshake is returned along with a // non-nil *http.Response so that callers can handle redirects, authentication, -// etc. +// etcetera. The response body may not contain the entire response and does not +// need to be closed by the application. func (d *Dialer) Dial(urlStr string, requestHeader http.Header) (*Conn, *http.Response, error) { + + if d == nil { + d = &Dialer{ + Proxy: http.ProxyFromEnvironment, + } + } + + challengeKey, err := generateChallengeKey() + if err != nil { + return nil, nil, err + } + u, err := parseURL(urlStr) if err != nil { return nil, nil, err } + switch u.Scheme { + case "ws": + u.Scheme = "http" + case "wss": + u.Scheme = "https" + default: + return nil, nil, errMalformedURL + } + + if u.User != nil { + // User name and password are not allowed in websocket URIs. + return nil, nil, errMalformedURL + } + + req := &http.Request{ + Method: "GET", + URL: u, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: make(http.Header), + Host: u.Host, + } + + // Set the cookies present in the cookie jar of the dialer + if d.Jar != nil { + for _, cookie := range d.Jar.Cookies(u) { + req.AddCookie(cookie) + } + } + + // Set the request headers using the capitalization for names and values in + // RFC examples. Although the capitalization shouldn't matter, there are + // servers that depend on it. The Header.Set method is not used because the + // method canonicalizes the header names. + req.Header["Upgrade"] = []string{"websocket"} + req.Header["Connection"] = []string{"Upgrade"} + req.Header["Sec-WebSocket-Key"] = []string{challengeKey} + req.Header["Sec-WebSocket-Version"] = []string{"13"} + if len(d.Subprotocols) > 0 { + req.Header["Sec-WebSocket-Protocol"] = []string{strings.Join(d.Subprotocols, ", ")} + } + for k, vs := range requestHeader { + switch { + case k == "Host": + if len(vs) > 0 { + req.Host = vs[0] + } + case k == "Upgrade" || + k == "Connection" || + k == "Sec-Websocket-Key" || + k == "Sec-Websocket-Version" || + k == "Sec-Websocket-Extensions" || + (k == "Sec-Websocket-Protocol" && len(d.Subprotocols) > 0): + return nil, nil, errors.New("websocket: duplicate header not allowed: " + k) + default: + req.Header[k] = vs + } + } + + if d.EnableCompression { + req.Header.Set("Sec-Websocket-Extensions", "permessage-deflate; server_no_context_takeover; client_no_context_takeover") + } + hostPort, hostNoPort := hostPortNoPort(u) - if d == nil { - d = &Dialer{} + var proxyURL *url.URL + // Check wether the proxy method has been configured + if d.Proxy != nil { + proxyURL, err = d.Proxy(req) + } + if err != nil { + return nil, nil, err + } + + var targetHostPort string + if proxyURL != nil { + targetHostPort, _ = hostPortNoPort(proxyURL) + } else { + targetHostPort = hostPort } var deadline time.Time @@ -179,7 +274,7 @@ func (d *Dialer) Dial(urlStr string, requestHeader http.Header) (*Conn, *http.Re netDial = netDialer.Dial } - netConn, err := netDial("tcp", hostPort) + netConn, err := netDial("tcp", targetHostPort) if err != nil { return nil, nil, err } @@ -194,13 +289,41 @@ func (d *Dialer) Dial(urlStr string, requestHeader http.Header) (*Conn, *http.Re return nil, nil, err } - if u.Scheme == "wss" { - cfg := d.TLSClientConfig - if cfg == nil { - cfg = &tls.Config{ServerName: hostNoPort} - } else if cfg.ServerName == "" { - shallowCopy := *cfg - cfg = &shallowCopy + if proxyURL != nil { + connectHeader := make(http.Header) + if user := proxyURL.User; user != nil { + proxyUser := user.Username() + if proxyPassword, passwordSet := user.Password(); passwordSet { + credential := base64.StdEncoding.EncodeToString([]byte(proxyUser + ":" + proxyPassword)) + connectHeader.Set("Proxy-Authorization", "Basic "+credential) + } + } + connectReq := &http.Request{ + Method: "CONNECT", + URL: &url.URL{Opaque: hostPort}, + Host: hostPort, + Header: connectHeader, + } + + connectReq.Write(netConn) + + // Read response. + // Okay to use and discard buffered reader here, because + // TLS server will not speak until spoken to. + br := bufio.NewReader(netConn) + resp, err := http.ReadResponse(br, connectReq) + if err != nil { + return nil, nil, err + } + if resp.StatusCode != 200 { + f := strings.SplitN(resp.Status, " ", 2) + return nil, nil, errors.New(f[1]) + } + } + + if u.Scheme == "https" { + cfg := cloneTLSConfig(d.TLSClientConfig) + if cfg.ServerName == "" { cfg.ServerName = hostNoPort } tlsConn := tls.Client(netConn, cfg) @@ -215,31 +338,83 @@ func (d *Dialer) Dial(urlStr string, requestHeader http.Header) (*Conn, *http.Re } } - readBufferSize := d.ReadBufferSize - if readBufferSize == 0 { - readBufferSize = 4096 + conn := newConn(netConn, false, d.ReadBufferSize, d.WriteBufferSize) + + if err := req.Write(netConn); err != nil { + return nil, nil, err } - writeBufferSize := d.WriteBufferSize - if writeBufferSize == 0 { - writeBufferSize = 4096 + resp, err := http.ReadResponse(conn.br, req) + if err != nil { + return nil, nil, err } - if len(d.Subprotocols) > 0 { - h := http.Header{} - for k, v := range requestHeader { - h[k] = v + if d.Jar != nil { + if rc := resp.Cookies(); len(rc) > 0 { + d.Jar.SetCookies(u, rc) } - h.Set("Sec-Websocket-Protocol", strings.Join(d.Subprotocols, ", ")) - requestHeader = h } - conn, resp, err := NewClient(netConn, u, requestHeader, readBufferSize, writeBufferSize) - if err != nil { - return nil, resp, err + if resp.StatusCode != 101 || + !strings.EqualFold(resp.Header.Get("Upgrade"), "websocket") || + !strings.EqualFold(resp.Header.Get("Connection"), "upgrade") || + resp.Header.Get("Sec-Websocket-Accept") != computeAcceptKey(challengeKey) { + // Before closing the network connection on return from this + // function, slurp up some of the response to aid application + // debugging. + buf := make([]byte, 1024) + n, _ := io.ReadFull(resp.Body, buf) + resp.Body = ioutil.NopCloser(bytes.NewReader(buf[:n])) + return nil, resp, ErrBadHandshake + } + + for _, ext := range parseExtensions(req.Header) { + if ext[""] != "permessage-deflate" { + continue + } + _, snct := ext["server_no_context_takeover"] + _, cnct := ext["client_no_context_takeover"] + if !snct || !cnct { + return nil, resp, errInvalidCompression + } + conn.newCompressionWriter = compressNoContextTakeover + conn.newDecompressionReader = decompressNoContextTakeover + break } + resp.Body = ioutil.NopCloser(bytes.NewReader([]byte{})) + conn.subprotocol = resp.Header.Get("Sec-Websocket-Protocol") + netConn.SetDeadline(time.Time{}) netConn = nil // to avoid close in defer. return conn, resp, nil } + +// cloneTLSConfig clones all public fields except the fields +// SessionTicketsDisabled and SessionTicketKey. This avoids copying the +// sync.Mutex in the sync.Once and makes it safe to call cloneTLSConfig on a +// config in active use. +func cloneTLSConfig(cfg *tls.Config) *tls.Config { + if cfg == nil { + return &tls.Config{} + } + return &tls.Config{ + Rand: cfg.Rand, + Time: cfg.Time, + Certificates: cfg.Certificates, + NameToCertificate: cfg.NameToCertificate, + GetCertificate: cfg.GetCertificate, + RootCAs: cfg.RootCAs, + NextProtos: cfg.NextProtos, + ServerName: cfg.ServerName, + ClientAuth: cfg.ClientAuth, + ClientCAs: cfg.ClientCAs, + InsecureSkipVerify: cfg.InsecureSkipVerify, + CipherSuites: cfg.CipherSuites, + PreferServerCipherSuites: cfg.PreferServerCipherSuites, + ClientSessionCache: cfg.ClientSessionCache, + MinVersion: cfg.MinVersion, + MaxVersion: cfg.MaxVersion, + CurvePreferences: cfg.CurvePreferences, + } +} diff --git a/agent/vendor/github.com/gorilla/websocket/compression.go b/agent/vendor/github.com/gorilla/websocket/compression.go new file mode 100644 index 00000000000..e2ac7617b42 --- /dev/null +++ b/agent/vendor/github.com/gorilla/websocket/compression.go @@ -0,0 +1,85 @@ +// Copyright 2016 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package websocket + +import ( + "compress/flate" + "errors" + "io" + "strings" +) + +func decompressNoContextTakeover(r io.Reader) io.Reader { + const tail = + // Add four bytes as specified in RFC + "\x00\x00\xff\xff" + + // Add final block to squelch unexpected EOF error from flate reader. + "\x01\x00\x00\xff\xff" + + return flate.NewReader(io.MultiReader(r, strings.NewReader(tail))) +} + +func compressNoContextTakeover(w io.WriteCloser) (io.WriteCloser, error) { + tw := &truncWriter{w: w} + fw, err := flate.NewWriter(tw, 3) + return &flateWrapper{fw: fw, tw: tw}, err +} + +// truncWriter is an io.Writer that writes all but the last four bytes of the +// stream to another io.Writer. +type truncWriter struct { + w io.WriteCloser + n int + p [4]byte +} + +func (w *truncWriter) Write(p []byte) (int, error) { + n := 0 + + // fill buffer first for simplicity. + if w.n < len(w.p) { + n = copy(w.p[w.n:], p) + p = p[n:] + w.n += n + if len(p) == 0 { + return n, nil + } + } + + m := len(p) + if m > len(w.p) { + m = len(w.p) + } + + if nn, err := w.w.Write(w.p[:m]); err != nil { + return n + nn, err + } + + copy(w.p[:], w.p[m:]) + copy(w.p[len(w.p)-m:], p[len(p)-m:]) + nn, err := w.w.Write(p[:len(p)-m]) + return n + nn, err +} + +type flateWrapper struct { + fw *flate.Writer + tw *truncWriter +} + +func (w *flateWrapper) Write(p []byte) (int, error) { + return w.fw.Write(p) +} + +func (w *flateWrapper) Close() error { + err1 := w.fw.Flush() + if w.tw.p != [4]byte{0, 0, 0xff, 0xff} { + return errors.New("websocket: internal error, unexpected bytes at end of flate stream") + } + err2 := w.tw.w.Close() + if err1 != nil { + return err1 + } + return err2 +} diff --git a/agent/vendor/github.com/gorilla/websocket/conn.go b/agent/vendor/github.com/gorilla/websocket/conn.go index 270114285ce..ce7f0a6159f 100644 --- a/agent/vendor/github.com/gorilla/websocket/conn.go +++ b/agent/vendor/github.com/gorilla/websocket/conn.go @@ -10,10 +10,33 @@ import ( "errors" "io" "io/ioutil" - "math/rand" "net" "strconv" + "sync" "time" + "unicode/utf8" +) + +const ( + // Frame header byte 0 bits from Section 5.2 of RFC 6455 + finalBit = 1 << 7 + rsv1Bit = 1 << 6 + rsv2Bit = 1 << 5 + rsv3Bit = 1 << 4 + + // Frame header byte 1 bits from Section 5.2 of RFC 6455 + maskBit = 1 << 7 + + maxFrameHeaderSize = 2 + 8 + 4 // Fixed header + length + mask + maxControlFramePayloadSize = 125 + + writeWait = time.Second + + defaultReadBufferSize = 4096 + defaultWriteBufferSize = 4096 + + continuationFrame = 0 + noFrame = -1 ) // Close codes defined in RFC 6455, section 11.7. @@ -29,6 +52,8 @@ const ( CloseMessageTooBig = 1009 CloseMandatoryExtension = 1010 CloseInternalServerErr = 1011 + CloseServiceRestart = 1012 + CloseTryAgainLater = 1013 CloseTLSHandshake = 1015 ) @@ -55,49 +80,109 @@ const ( PongMessage = 10 ) -var ( - continuationFrame = 0 - noFrame = -1 -) - -var ( - // ErrCloseSent is returned when the application writes a message to the - // connection after sending a close message. - ErrCloseSent = errors.New("websocket: close sent") +// ErrCloseSent is returned when the application writes a message to the +// connection after sending a close message. +var ErrCloseSent = errors.New("websocket: close sent") - // ErrReadLimit is returned when reading a message that is larger than the - // read limit set for the connection. - ErrReadLimit = errors.New("websocket: read limit exceeded") -) +// ErrReadLimit is returned when reading a message that is larger than the +// read limit set for the connection. +var ErrReadLimit = errors.New("websocket: read limit exceeded") -type websocketError struct { +// netError satisfies the net Error interface. +type netError struct { msg string temporary bool timeout bool } -func (e *websocketError) Error() string { return e.msg } -func (e *websocketError) Temporary() bool { return e.temporary } -func (e *websocketError) Timeout() bool { return e.timeout } +func (e *netError) Error() string { return e.msg } +func (e *netError) Temporary() bool { return e.temporary } +func (e *netError) Timeout() bool { return e.timeout } + +// CloseError represents close frame. +type CloseError struct { + + // Code is defined in RFC 6455, section 11.7. + Code int + + // Text is the optional text payload. + Text string +} + +func (e *CloseError) Error() string { + s := []byte("websocket: close ") + s = strconv.AppendInt(s, int64(e.Code), 10) + switch e.Code { + case CloseNormalClosure: + s = append(s, " (normal)"...) + case CloseGoingAway: + s = append(s, " (going away)"...) + case CloseProtocolError: + s = append(s, " (protocol error)"...) + case CloseUnsupportedData: + s = append(s, " (unsupported data)"...) + case CloseNoStatusReceived: + s = append(s, " (no status)"...) + case CloseAbnormalClosure: + s = append(s, " (abnormal closure)"...) + case CloseInvalidFramePayloadData: + s = append(s, " (invalid payload data)"...) + case ClosePolicyViolation: + s = append(s, " (policy violation)"...) + case CloseMessageTooBig: + s = append(s, " (message too big)"...) + case CloseMandatoryExtension: + s = append(s, " (mandatory extension missing)"...) + case CloseInternalServerErr: + s = append(s, " (internal server error)"...) + case CloseTLSHandshake: + s = append(s, " (TLS handshake error)"...) + } + if e.Text != "" { + s = append(s, ": "...) + s = append(s, e.Text...) + } + return string(s) +} + +// IsCloseError returns boolean indicating whether the error is a *CloseError +// with one of the specified codes. +func IsCloseError(err error, codes ...int) bool { + if e, ok := err.(*CloseError); ok { + for _, code := range codes { + if e.Code == code { + return true + } + } + } + return false +} + +// IsUnexpectedCloseError returns boolean indicating whether the error is a +// *CloseError with a code not in the list of expected codes. +func IsUnexpectedCloseError(err error, expectedCodes ...int) bool { + if e, ok := err.(*CloseError); ok { + for _, code := range expectedCodes { + if e.Code == code { + return false + } + } + return true + } + return false +} var ( - errWriteTimeout = &websocketError{msg: "websocket: write timeout", timeout: true} + errWriteTimeout = &netError{msg: "websocket: write timeout", timeout: true, temporary: true} + errUnexpectedEOF = &CloseError{Code: CloseAbnormalClosure, Text: io.ErrUnexpectedEOF.Error()} errBadWriteOpCode = errors.New("websocket: bad write message type") errWriteClosed = errors.New("websocket: write closed") errInvalidControlFrame = errors.New("websocket: invalid control frame") ) -const ( - maxFrameHeaderSize = 2 + 8 + 4 // Fixed header + length + mask - maxControlFramePayloadSize = 125 - finalBit = 1 << 7 - maskBit = 1 << 7 - writeWait = time.Second -) - func hideTempErr(err error) error { if e, ok := err.(net.Error); ok && e.Temporary() { - err = struct{ error }{err} + err = &netError{msg: e.Error(), timeout: e.Timeout()} } return err } @@ -110,65 +195,91 @@ func isData(frameType int) bool { return frameType == TextMessage || frameType == BinaryMessage } -func maskBytes(key [4]byte, pos int, b []byte) int { - for i := range b { - b[i] ^= key[pos&3] - pos++ - } - return pos & 3 +var validReceivedCloseCodes = map[int]bool{ + // see http://www.iana.org/assignments/websocket/websocket.xhtml#close-code-number + + CloseNormalClosure: true, + CloseGoingAway: true, + CloseProtocolError: true, + CloseUnsupportedData: true, + CloseNoStatusReceived: false, + CloseAbnormalClosure: false, + CloseInvalidFramePayloadData: true, + ClosePolicyViolation: true, + CloseMessageTooBig: true, + CloseMandatoryExtension: true, + CloseInternalServerErr: true, + CloseServiceRestart: true, + CloseTryAgainLater: true, + CloseTLSHandshake: false, } -func newMaskKey() [4]byte { - n := rand.Uint32() - return [4]byte{byte(n), byte(n >> 8), byte(n >> 16), byte(n >> 24)} +func isValidReceivedCloseCode(code int) bool { + return validReceivedCloseCodes[code] || (code >= 3000 && code <= 4999) } -// Conn represents a WebSocket connection. +// The Conn type represents a WebSocket connection. type Conn struct { conn net.Conn isServer bool subprotocol string // Write fields - mu chan bool // used as mutex to protect write to conn and closeSent - closeSent bool // true if close message was sent + mu chan bool // used as mutex to protect write to conn + writeBuf []byte // frame is constructed in this buffer. + writeDeadline time.Time + writer io.WriteCloser // the current writer returned to the application + isWriting bool // for best-effort concurrent write detection - // Message writer fields. - writeErr error - writeBuf []byte // frame is constructed in this buffer. - writePos int // end of data in writeBuf. - writeFrameType int // type of the current frame. - writeSeq int // incremented to invalidate message writers. - writeDeadline time.Time + writeErrMu sync.Mutex + writeErr error + + enableWriteCompression bool + newCompressionWriter func(io.WriteCloser) (io.WriteCloser, error) // Read fields readErr error br *bufio.Reader readRemaining int64 // bytes remaining in current frame. readFinal bool // true the current message has more frames. - readSeq int // incremented to invalidate message readers. readLength int64 // Message size. readLimit int64 // Maximum message size. readMaskPos int readMaskKey [4]byte handlePong func(string) error handlePing func(string) error + handleClose func(int, string) error + readErrCount int + messageReader *messageReader // the current low-level reader + + readDecompress bool // whether last read frame had RSV1 set + newDecompressionReader func(io.Reader) io.Reader } -func newConn(conn net.Conn, isServer bool, readBufSize, writeBufSize int) *Conn { +func newConn(conn net.Conn, isServer bool, readBufferSize, writeBufferSize int) *Conn { mu := make(chan bool, 1) mu <- true + if readBufferSize == 0 { + readBufferSize = defaultReadBufferSize + } + if readBufferSize < maxControlFramePayloadSize { + readBufferSize = maxControlFramePayloadSize + } + if writeBufferSize == 0 { + writeBufferSize = defaultWriteBufferSize + } + c := &Conn{ - isServer: isServer, - br: bufio.NewReaderSize(conn, readBufSize), - conn: conn, - mu: mu, - readFinal: true, - writeBuf: make([]byte, writeBufSize+maxFrameHeaderSize), - writeFrameType: noFrame, - writePos: maxFrameHeaderSize, + isServer: isServer, + br: bufio.NewReaderSize(conn, readBufferSize), + conn: conn, + mu: mu, + readFinal: true, + writeBuf: make([]byte, writeBufferSize+maxFrameHeaderSize), + enableWriteCompression: true, } + c.SetCloseHandler(nil) c.SetPingHandler(nil) c.SetPongHandler(nil) return c @@ -196,29 +307,40 @@ func (c *Conn) RemoteAddr() net.Addr { // Write methods +func (c *Conn) writeFatal(err error) error { + err = hideTempErr(err) + c.writeErrMu.Lock() + if c.writeErr == nil { + c.writeErr = err + } + c.writeErrMu.Unlock() + return err +} + func (c *Conn) write(frameType int, deadline time.Time, bufs ...[]byte) error { <-c.mu defer func() { c.mu <- true }() - if c.closeSent { - return ErrCloseSent - } else if frameType == CloseMessage { - c.closeSent = true + c.writeErrMu.Lock() + err := c.writeErr + c.writeErrMu.Unlock() + if err != nil { + return err } c.conn.SetWriteDeadline(deadline) for _, buf := range bufs { if len(buf) > 0 { - n, err := c.conn.Write(buf) - if n != len(buf) { - // Close on partial write. - c.conn.Close() - } + _, err := c.conn.Write(buf) if err != nil { - return err + return c.writeFatal(err) } } } + + if frameType == CloseMessage { + c.writeFatal(ErrCloseSent) + } return nil } @@ -267,63 +389,108 @@ func (c *Conn) WriteControl(messageType int, data []byte, deadline time.Time) er } defer func() { c.mu <- true }() - if c.closeSent { - return ErrCloseSent - } else if messageType == CloseMessage { - c.closeSent = true + c.writeErrMu.Lock() + err := c.writeErr + c.writeErrMu.Unlock() + if err != nil { + return err } c.conn.SetWriteDeadline(deadline) - n, err := c.conn.Write(buf) - if n != 0 && n != len(buf) { - c.conn.Close() + _, err = c.conn.Write(buf) + if err != nil { + return c.writeFatal(err) + } + if messageType == CloseMessage { + c.writeFatal(ErrCloseSent) } return err } -// NextWriter returns a writer for the next message to send. The writer's -// Close method flushes the complete message to the network. +func (c *Conn) prepWrite(messageType int) error { + // Close previous writer if not already closed by the application. It's + // probably better to return an error in this situation, but we cannot + // change this without breaking existing applications. + if c.writer != nil { + c.writer.Close() + c.writer = nil + } + + if !isControl(messageType) && !isData(messageType) { + return errBadWriteOpCode + } + + c.writeErrMu.Lock() + err := c.writeErr + c.writeErrMu.Unlock() + return err +} + +// NextWriter returns a writer for the next message to send. The writer's Close +// method flushes the complete message to the network. // // There can be at most one open writer on a connection. NextWriter closes the // previous writer if the application has not already done so. -// -// The NextWriter method and the writers returned from the method cannot be -// accessed by more than one goroutine at a time. func (c *Conn) NextWriter(messageType int) (io.WriteCloser, error) { - if c.writeErr != nil { - return nil, c.writeErr + if err := c.prepWrite(messageType); err != nil { + return nil, err } - if c.writeFrameType != noFrame { - if err := c.flushFrame(true, nil); err != nil { + mw := &messageWriter{ + c: c, + frameType: messageType, + pos: maxFrameHeaderSize, + } + c.writer = mw + if c.newCompressionWriter != nil && c.enableWriteCompression && isData(messageType) { + w, err := c.newCompressionWriter(c.writer) + if err != nil { + c.writer = nil return nil, err } + mw.compress = true + c.writer = w } + return c.writer, nil +} - if !isControl(messageType) && !isData(messageType) { - return nil, errBadWriteOpCode - } +type messageWriter struct { + c *Conn + compress bool // whether next call to flushFrame should set RSV1 + pos int // end of data in writeBuf. + frameType int // type of the current frame. + err error +} - c.writeFrameType = messageType - return messageWriter{c, c.writeSeq}, nil +func (w *messageWriter) fatal(err error) error { + if w.err != nil { + w.err = err + w.c.writer = nil + } + return err } -func (c *Conn) flushFrame(final bool, extra []byte) error { - length := c.writePos - maxFrameHeaderSize + len(extra) +// flushFrame writes buffered data and extra as a frame to the network. The +// final argument indicates that this is the last frame in the message. +func (w *messageWriter) flushFrame(final bool, extra []byte) error { + c := w.c + length := w.pos - maxFrameHeaderSize + len(extra) // Check for invalid control frames. - if isControl(c.writeFrameType) && + if isControl(w.frameType) && (!final || length > maxControlFramePayloadSize) { - c.writeSeq++ - c.writeFrameType = noFrame - c.writePos = maxFrameHeaderSize - return errInvalidControlFrame + return w.fatal(errInvalidControlFrame) } - b0 := byte(c.writeFrameType) + b0 := byte(w.frameType) if final { b0 |= finalBit } + if w.compress { + b0 |= rsv1Bit + } + w.compress = false + b1 := byte(0) if !c.isServer { b1 |= maskBit @@ -355,49 +522,50 @@ func (c *Conn) flushFrame(final bool, extra []byte) error { if !c.isServer { key := newMaskKey() copy(c.writeBuf[maxFrameHeaderSize-4:], key[:]) - maskBytes(key, 0, c.writeBuf[maxFrameHeaderSize:c.writePos]) + maskBytes(key, 0, c.writeBuf[maxFrameHeaderSize:w.pos]) if len(extra) > 0 { - c.writeErr = errors.New("websocket: internal error, extra used in client mode") - return c.writeErr + return c.writeFatal(errors.New("websocket: internal error, extra used in client mode")) } } - // Write the buffers to the connection. - c.writeErr = c.write(c.writeFrameType, c.writeDeadline, c.writeBuf[framePos:c.writePos], extra) + // Write the buffers to the connection with best-effort detection of + // concurrent writes. See the concurrency section in the package + // documentation for more info. - // Setup for next frame. - c.writePos = maxFrameHeaderSize - c.writeFrameType = continuationFrame - if final { - c.writeSeq++ - c.writeFrameType = noFrame + if c.isWriting { + panic("concurrent write to websocket connection") } - return c.writeErr -} + c.isWriting = true -type messageWriter struct { - c *Conn - seq int -} + err := c.write(w.frameType, c.writeDeadline, c.writeBuf[framePos:w.pos], extra) -func (w messageWriter) err() error { - c := w.c - if c.writeSeq != w.seq { - return errWriteClosed + if !c.isWriting { + panic("concurrent write to websocket connection") + } + c.isWriting = false + + if err != nil { + return w.fatal(err) } - if c.writeErr != nil { - return c.writeErr + + if final { + c.writer = nil + return nil } + + // Setup for next frame. + w.pos = maxFrameHeaderSize + w.frameType = continuationFrame return nil } -func (w messageWriter) ncopy(max int) (int, error) { - n := len(w.c.writeBuf) - w.c.writePos +func (w *messageWriter) ncopy(max int) (int, error) { + n := len(w.c.writeBuf) - w.pos if n <= 0 { - if err := w.c.flushFrame(false, nil); err != nil { + if err := w.flushFrame(false, nil); err != nil { return 0, err } - n = len(w.c.writeBuf) - w.c.writePos + n = len(w.c.writeBuf) - w.pos } if n > max { n = max @@ -405,14 +573,14 @@ func (w messageWriter) ncopy(max int) (int, error) { return n, nil } -func (w messageWriter) write(final bool, p []byte) (int, error) { - if err := w.err(); err != nil { - return 0, err +func (w *messageWriter) Write(p []byte) (int, error) { + if w.err != nil { + return 0, w.err } if len(p) > 2*len(w.c.writeBuf) && w.c.isServer { // Don't buffer large messages. - err := w.c.flushFrame(final, p) + err := w.flushFrame(false, p) if err != nil { return 0, err } @@ -425,20 +593,16 @@ func (w messageWriter) write(final bool, p []byte) (int, error) { if err != nil { return 0, err } - copy(w.c.writeBuf[w.c.writePos:], p[:n]) - w.c.writePos += n + copy(w.c.writeBuf[w.pos:], p[:n]) + w.pos += n p = p[n:] } return nn, nil } -func (w messageWriter) Write(p []byte) (int, error) { - return w.write(false, p) -} - -func (w messageWriter) WriteString(p string) (int, error) { - if err := w.err(); err != nil { - return 0, err +func (w *messageWriter) WriteString(p string) (int, error) { + if w.err != nil { + return 0, w.err } nn := len(p) @@ -447,27 +611,27 @@ func (w messageWriter) WriteString(p string) (int, error) { if err != nil { return 0, err } - copy(w.c.writeBuf[w.c.writePos:], p[:n]) - w.c.writePos += n + copy(w.c.writeBuf[w.pos:], p[:n]) + w.pos += n p = p[n:] } return nn, nil } -func (w messageWriter) ReadFrom(r io.Reader) (nn int64, err error) { - if err := w.err(); err != nil { - return 0, err +func (w *messageWriter) ReadFrom(r io.Reader) (nn int64, err error) { + if w.err != nil { + return 0, w.err } for { - if w.c.writePos == len(w.c.writeBuf) { - err = w.c.flushFrame(false, nil) + if w.pos == len(w.c.writeBuf) { + err = w.flushFrame(false, nil) if err != nil { break } } var n int - n, err = r.Read(w.c.writeBuf[w.c.writePos:]) - w.c.writePos += n + n, err = r.Read(w.c.writeBuf[w.pos:]) + w.pos += n nn += int64(n) if err != nil { if err == io.EOF { @@ -479,36 +643,49 @@ func (w messageWriter) ReadFrom(r io.Reader) (nn int64, err error) { return nn, err } -func (w messageWriter) Close() error { - if err := w.err(); err != nil { +func (w *messageWriter) Close() error { + if w.err != nil { + return w.err + } + if err := w.flushFrame(true, nil); err != nil { return err } - return w.c.flushFrame(true, nil) + w.err = errWriteClosed + return nil } // WriteMessage is a helper method for getting a writer using NextWriter, // writing the message and closing the writer. func (c *Conn) WriteMessage(messageType int, data []byte) error { - wr, err := c.NextWriter(messageType) + + if c.isServer && (c.newCompressionWriter == nil || !c.enableWriteCompression) { + + // Fast path with no allocations and single frame. + + if err := c.prepWrite(messageType); err != nil { + return err + } + mw := messageWriter{c: c, frameType: messageType, pos: maxFrameHeaderSize} + n := copy(c.writeBuf[mw.pos:], data) + mw.pos += n + data = data[n:] + return mw.flushFrame(true, data) + } + + w, err := c.NextWriter(messageType) if err != nil { return err } - w := wr.(messageWriter) - if _, err := w.write(true, data); err != nil { + if _, err = w.Write(data); err != nil { return err } - if c.writeSeq == w.seq { - if err := c.flushFrame(true, nil); err != nil { - return err - } - } - return nil + return w.Close() } // SetWriteDeadline sets the write deadline on the underlying network // connection. After a write has timed out, the websocket state is corrupt and // all future writes will return an error. A zero value for t means writes will -// not time out +// not time out. func (c *Conn) SetWriteDeadline(t time.Time) error { c.writeDeadline = t return nil @@ -516,22 +693,6 @@ func (c *Conn) SetWriteDeadline(t time.Time) error { // Read methods -// readFull is like io.ReadFull except that io.EOF is never returned. -func (c *Conn) readFull(p []byte) (err error) { - var n int - for n < len(p) && err == nil { - var nn int - nn, err = c.br.Read(p[n:]) - n += nn - } - if n == len(p) { - err = nil - } else if err == io.EOF { - err = io.ErrUnexpectedEOF - } - return -} - func (c *Conn) advanceFrame() (int, error) { // 1. Skip remainder of previous frame. @@ -544,19 +705,24 @@ func (c *Conn) advanceFrame() (int, error) { // 2. Read and parse first two bytes of frame header. - var b [8]byte - if err := c.readFull(b[:2]); err != nil { + p, err := c.read(2) + if err != nil { return noFrame, err } - final := b[0]&finalBit != 0 - frameType := int(b[0] & 0xf) - reserved := int((b[0] >> 4) & 0x7) - mask := b[1]&maskBit != 0 - c.readRemaining = int64(b[1] & 0x7f) + final := p[0]&finalBit != 0 + frameType := int(p[0] & 0xf) + mask := p[1]&maskBit != 0 + c.readRemaining = int64(p[1] & 0x7f) - if reserved != 0 { - return noFrame, c.handleProtocolError("unexpected reserved bits " + strconv.Itoa(reserved)) + c.readDecompress = false + if c.newDecompressionReader != nil && (p[0]&rsv1Bit) != 0 { + c.readDecompress = true + p[0] &^= rsv1Bit + } + + if rsv := p[0] & (rsv1Bit | rsv2Bit | rsv3Bit); rsv != 0 { + return noFrame, c.handleProtocolError("unexpected reserved bits 0x" + strconv.FormatInt(int64(rsv), 16)) } switch frameType { @@ -585,15 +751,17 @@ func (c *Conn) advanceFrame() (int, error) { switch c.readRemaining { case 126: - if err := c.readFull(b[:2]); err != nil { + p, err := c.read(2) + if err != nil { return noFrame, err } - c.readRemaining = int64(binary.BigEndian.Uint16(b[:2])) + c.readRemaining = int64(binary.BigEndian.Uint16(p)) case 127: - if err := c.readFull(b[:8]); err != nil { + p, err := c.read(8) + if err != nil { return noFrame, err } - c.readRemaining = int64(binary.BigEndian.Uint64(b[:8])) + c.readRemaining = int64(binary.BigEndian.Uint64(p)) } // 4. Handle frame masking. @@ -604,9 +772,11 @@ func (c *Conn) advanceFrame() (int, error) { if mask { c.readMaskPos = 0 - if err := c.readFull(c.readMaskKey[:]); err != nil { + p, err := c.read(len(c.readMaskKey)) + if err != nil { return noFrame, err } + copy(c.readMaskKey[:], p) } // 5. For text and binary messages, enforce read limit and return. @@ -626,9 +796,9 @@ func (c *Conn) advanceFrame() (int, error) { var payload []byte if c.readRemaining > 0 { - payload = make([]byte, c.readRemaining) + payload, err = c.read(int(c.readRemaining)) c.readRemaining = 0 - if err := c.readFull(payload); err != nil { + if err != nil { return noFrame, err } if c.isServer { @@ -648,19 +818,22 @@ func (c *Conn) advanceFrame() (int, error) { return noFrame, err } case CloseMessage: - c.WriteControl(CloseMessage, []byte{}, time.Now().Add(writeWait)) - if len(payload) < 2 { - return noFrame, io.EOF + closeCode := CloseNoStatusReceived + closeText := "" + if len(payload) >= 2 { + closeCode = int(binary.BigEndian.Uint16(payload)) + if !isValidReceivedCloseCode(closeCode) { + return noFrame, c.handleProtocolError("invalid close code") + } + closeText = string(payload[2:]) + if !utf8.ValidString(closeText) { + return noFrame, c.handleProtocolError("invalid utf8 payload in close frame") + } } - closeCode := binary.BigEndian.Uint16(payload) - switch closeCode { - case CloseNormalClosure, CloseGoingAway: - return noFrame, io.EOF - default: - return noFrame, errors.New("websocket: close " + - strconv.Itoa(int(closeCode)) + " " + - string(payload[2:])) + if err := c.handleClose(closeCode, closeText); err != nil { + return noFrame, err } + return noFrame, &CloseError{Code: closeCode, Text: closeText} } return frameType, nil @@ -677,11 +850,13 @@ func (c *Conn) handleProtocolError(message string) error { // There can be at most one open reader on a connection. NextReader discards // the previous message if the application has not already consumed it. // -// The NextReader method and the readers returned from the method cannot be -// accessed by more than one goroutine at a time. +// Applications must break out of the application's read loop when this method +// returns a non-nil error value. Errors returned from this method are +// permanent. Once this method returns a non-nil error, all subsequent calls to +// this method return the same error. func (c *Conn) NextReader() (messageType int, r io.Reader, err error) { - c.readSeq++ + c.messageReader = nil c.readLength = 0 for c.readErr == nil { @@ -691,55 +866,69 @@ func (c *Conn) NextReader() (messageType int, r io.Reader, err error) { break } if frameType == TextMessage || frameType == BinaryMessage { - return frameType, messageReader{c, c.readSeq}, nil + c.messageReader = &messageReader{c} + var r io.Reader = c.messageReader + if c.readDecompress { + r = c.newDecompressionReader(r) + } + return frameType, r, nil } } - return noFrame, nil, c.readErr -} -type messageReader struct { - c *Conn - seq int + // Applications that do handle the error returned from this method spin in + // tight loop on connection failure. To help application developers detect + // this error, panic on repeated reads to the failed connection. + c.readErrCount++ + if c.readErrCount >= 1000 { + panic("repeated read on failed websocket connection") + } + + return noFrame, nil, c.readErr } -func (r messageReader) Read(b []byte) (int, error) { +type messageReader struct{ c *Conn } - if r.seq != r.c.readSeq { +func (r *messageReader) Read(b []byte) (int, error) { + c := r.c + if c.messageReader != r { return 0, io.EOF } - for r.c.readErr == nil { + for c.readErr == nil { - if r.c.readRemaining > 0 { - if int64(len(b)) > r.c.readRemaining { - b = b[:r.c.readRemaining] + if c.readRemaining > 0 { + if int64(len(b)) > c.readRemaining { + b = b[:c.readRemaining] + } + n, err := c.br.Read(b) + c.readErr = hideTempErr(err) + if c.isServer { + c.readMaskPos = maskBytes(c.readMaskKey, c.readMaskPos, b[:n]) } - n, err := r.c.br.Read(b) - r.c.readErr = hideTempErr(err) - if r.c.isServer { - r.c.readMaskPos = maskBytes(r.c.readMaskKey, r.c.readMaskPos, b[:n]) + c.readRemaining -= int64(n) + if c.readRemaining > 0 && c.readErr == io.EOF { + c.readErr = errUnexpectedEOF } - r.c.readRemaining -= int64(n) - return n, r.c.readErr + return n, c.readErr } - if r.c.readFinal { - r.c.readSeq++ + if c.readFinal { + c.messageReader = nil return 0, io.EOF } - frameType, err := r.c.advanceFrame() + frameType, err := c.advanceFrame() switch { case err != nil: - r.c.readErr = hideTempErr(err) + c.readErr = hideTempErr(err) case frameType == TextMessage || frameType == BinaryMessage: - r.c.readErr = errors.New("websocket: internal error, unexpected text or binary in Reader") + c.readErr = errors.New("websocket: internal error, unexpected text or binary in Reader") } } - err := r.c.readErr - if err == io.EOF && r.seq == r.c.readSeq { - err = io.ErrUnexpectedEOF + err := c.readErr + if err == io.EOF && c.messageReader == r { + err = errUnexpectedEOF } return 0, err } @@ -771,21 +960,61 @@ func (c *Conn) SetReadLimit(limit int64) { c.readLimit = limit } +// CloseHandler returns the current close handler +func (c *Conn) CloseHandler() func(code int, text string) error { + return c.handleClose +} + +// SetCloseHandler sets the handler for close messages received from the peer. +// The code argument to h is the received close code or CloseNoStatusReceived +// if the close message is empty. The default close handler sends a close frame +// back to the peer. +func (c *Conn) SetCloseHandler(h func(code int, text string) error) { + if h == nil { + h = func(code int, text string) error { + message := []byte{} + if code != CloseNoStatusReceived { + message = FormatCloseMessage(code, "") + } + c.WriteControl(CloseMessage, message, time.Now().Add(writeWait)) + return nil + } + } + c.handleClose = h +} + +// PingHandler returns the current ping handler +func (c *Conn) PingHandler() func(appData string) error { + return c.handlePing +} + // SetPingHandler sets the handler for ping messages received from the peer. -// The default ping handler sends a pong to the peer. -func (c *Conn) SetPingHandler(h func(string) error) { +// The appData argument to h is the PING frame application data. The default +// ping handler sends a pong to the peer. +func (c *Conn) SetPingHandler(h func(appData string) error) { if h == nil { h = func(message string) error { - c.WriteControl(PongMessage, []byte(message), time.Now().Add(writeWait)) - return nil + err := c.WriteControl(PongMessage, []byte(message), time.Now().Add(writeWait)) + if err == ErrCloseSent { + return nil + } else if e, ok := err.(net.Error); ok && e.Temporary() { + return nil + } + return err } } c.handlePing = h } -// SetPongHandler sets then handler for pong messages received from the peer. -// The default pong handler does nothing. -func (c *Conn) SetPongHandler(h func(string) error) { +// PongHandler returns the current pong handler +func (c *Conn) PongHandler() func(appData string) error { + return c.handlePong +} + +// SetPongHandler sets the handler for pong messages received from the peer. +// The appData argument to h is the PONG frame application data. The default +// pong handler does nothing. +func (c *Conn) SetPongHandler(h func(appData string) error) { if h == nil { h = func(string) error { return nil } } @@ -798,6 +1027,13 @@ func (c *Conn) UnderlyingConn() net.Conn { return c.conn } +// EnableWriteCompression enables and disables write compression of +// subsequent text and binary messages. This function is a noop if +// compression was not negotiated with the peer. +func (c *Conn) EnableWriteCompression(enable bool) { + c.enableWriteCompression = enable +} + // FormatCloseMessage formats closeCode and text as a WebSocket close message. func FormatCloseMessage(closeCode int, text string) []byte { buf := make([]byte, 2+len(text)) diff --git a/agent/vendor/github.com/gorilla/websocket/conn_read.go b/agent/vendor/github.com/gorilla/websocket/conn_read.go new file mode 100644 index 00000000000..1ea15059ee1 --- /dev/null +++ b/agent/vendor/github.com/gorilla/websocket/conn_read.go @@ -0,0 +1,18 @@ +// Copyright 2016 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.5 + +package websocket + +import "io" + +func (c *Conn) read(n int) ([]byte, error) { + p, err := c.br.Peek(n) + if err == io.EOF { + err = errUnexpectedEOF + } + c.br.Discard(len(p)) + return p, err +} diff --git a/agent/vendor/github.com/gorilla/websocket/conn_read_legacy.go b/agent/vendor/github.com/gorilla/websocket/conn_read_legacy.go new file mode 100644 index 00000000000..018541cf6cb --- /dev/null +++ b/agent/vendor/github.com/gorilla/websocket/conn_read_legacy.go @@ -0,0 +1,21 @@ +// Copyright 2016 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !go1.5 + +package websocket + +import "io" + +func (c *Conn) read(n int) ([]byte, error) { + p, err := c.br.Peek(n) + if err == io.EOF { + err = errUnexpectedEOF + } + if len(p) > 0 { + // advance over the bytes just read + io.ReadFull(c.br, p) + } + return p, err +} diff --git a/agent/vendor/github.com/gorilla/websocket/doc.go b/agent/vendor/github.com/gorilla/websocket/doc.go index 798de9c3a51..610acf71289 100644 --- a/agent/vendor/github.com/gorilla/websocket/doc.go +++ b/agent/vendor/github.com/gorilla/websocket/doc.go @@ -24,7 +24,7 @@ // ... Use conn to send and receive messages. // } // -// Call the connection WriteMessage and ReadMessages methods to send and +// Call the connection's WriteMessage and ReadMessage methods to send and // receive messages as a slice of bytes. This snippet of code shows how to echo // messages using these methods: // @@ -46,8 +46,7 @@ // method to get an io.WriteCloser, write the message to the writer and close // the writer when done. To receive a message, call the connection NextReader // method to get an io.Reader and read until io.EOF is returned. This snippet -// snippet shows how to echo messages using the NextWriter and NextReader -// methods: +// shows how to echo messages using the NextWriter and NextReader methods: // // for { // messageType, r, err := conn.NextReader() @@ -86,28 +85,23 @@ // and pong. Call the connection WriteControl, WriteMessage or NextWriter // methods to send a control message to the peer. // -// Connections handle received ping and pong messages by invoking a callback -// function set with SetPingHandler and SetPongHandler methods. These callback -// functions can be invoked from the ReadMessage method, the NextReader method -// or from a call to the data message reader returned from NextReader. +// Connections handle received close messages by sending a close message to the +// peer and returning a *CloseError from the the NextReader, ReadMessage or the +// message Read method. // -// Connections handle received close messages by returning an error from the -// ReadMessage method, the NextReader method or from a call to the data message -// reader returned from NextReader. +// Connections handle received ping and pong messages by invoking callback +// functions set with SetPingHandler and SetPongHandler methods. The callback +// functions are called from the NextReader, ReadMessage and the message Read +// methods. // -// Concurrency -// -// A Conn supports a single concurrent caller to the write methods (NextWriter, -// SetWriteDeadline, WriteMessage) and a single concurrent caller to the read -// methods (NextReader, SetReadDeadline, ReadMessage). The Close and -// WriteControl methods can be called concurrently with all other methods. +// The default ping handler sends a pong to the peer. The application's reading +// goroutine can block for a short time while the handler writes the pong data +// to the connection. // -// Read is Required -// -// The application must read the connection to process ping and close messages -// sent from the peer. If the application is not otherwise interested in -// messages from the peer, then the application should start a goroutine to read -// and discard messages from the peer. A simple example is: +// The application must read the connection to process ping, pong and close +// messages sent from the peer. If the application is not otherwise interested +// in messages from the peer, then the application should start a goroutine to +// read and discard messages from the peer. A simple example is: // // func readLoop(c *websocket.Conn) { // for { @@ -118,6 +112,19 @@ // } // } // +// Concurrency +// +// Connections support one concurrent reader and one concurrent writer. +// +// Applications are responsible for ensuring that no more than one goroutine +// calls the write methods (NextWriter, SetWriteDeadline, WriteMessage, +// WriteJSON) concurrently and that no more than one goroutine calls the read +// methods (NextReader, SetReadDeadline, ReadMessage, ReadJSON, SetPongHandler, +// SetPingHandler) concurrently. +// +// The Close and WriteControl methods can be called concurrently with all other +// methods. +// // Origin Considerations // // Web browsers allow Javascript applications to open a WebSocket connection to @@ -135,11 +142,32 @@ // An application can allow connections from any origin by specifying a // function that always returns true: // -// var upgrader = websocket.Upgrader{ +// var upgrader = websocket.Upgrader{ // CheckOrigin: func(r *http.Request) bool { return true }, -// } +// } // -// The deprecated Upgrade function does enforce an origin policy. It's the +// The deprecated Upgrade function does not enforce an origin policy. It's the // application's responsibility to check the Origin header before calling // Upgrade. +// +// Compression [Experimental] +// +// Per message compression extensions (RFC 7692) are experimentally supported +// by this package in a limited capacity. Setting the EnableCompression option +// to true in Dialer or Upgrader will attempt to negotiate per message deflate +// support. If compression was successfully negotiated with the connection's +// peer, any message received in compressed form will be automatically +// decompressed. All Read methods will return uncompressed bytes. +// +// Per message compression of messages written to a connection can be enabled +// or disabled by calling the corresponding Conn method: +// +// conn.EnableWriteCompression(true) +// +// Currently this package does not support compression with "context takeover". +// This means that messages must be compressed and decompressed in isolation, +// without retaining sliding window or dictionary state across messages. For +// more details refer to RFC 7692. +// +// Use of compression is experimental and may result in decreased performance. package websocket diff --git a/agent/vendor/github.com/gorilla/websocket/json.go b/agent/vendor/github.com/gorilla/websocket/json.go index e0668f25e15..4f0e36875a5 100644 --- a/agent/vendor/github.com/gorilla/websocket/json.go +++ b/agent/vendor/github.com/gorilla/websocket/json.go @@ -6,6 +6,7 @@ package websocket import ( "encoding/json" + "io" ) // WriteJSON is deprecated, use c.WriteJSON instead. @@ -45,5 +46,10 @@ func (c *Conn) ReadJSON(v interface{}) error { if err != nil { return err } - return json.NewDecoder(r).Decode(v) + err = json.NewDecoder(r).Decode(v) + if err == io.EOF { + // One value is expected in the message. + err = io.ErrUnexpectedEOF + } + return err } diff --git a/agent/vendor/github.com/gorilla/websocket/mask.go b/agent/vendor/github.com/gorilla/websocket/mask.go new file mode 100644 index 00000000000..6758a2cb7a4 --- /dev/null +++ b/agent/vendor/github.com/gorilla/websocket/mask.go @@ -0,0 +1,61 @@ +// Copyright 2016 The Gorilla WebSocket Authors. All rights reserved. Use of +// this source code is governed by a BSD-style license that can be found in the +// LICENSE file. + +package websocket + +import ( + "math/rand" + "unsafe" +) + +const wordSize = int(unsafe.Sizeof(uintptr(0))) + +func newMaskKey() [4]byte { + n := rand.Uint32() + return [4]byte{byte(n), byte(n >> 8), byte(n >> 16), byte(n >> 24)} +} + +func maskBytes(key [4]byte, pos int, b []byte) int { + + // Mask one byte at a time for small buffers. + if len(b) < 2*wordSize { + for i := range b { + b[i] ^= key[pos&3] + pos++ + } + return pos & 3 + } + + // Mask one byte at a time to word boundary. + if n := int(uintptr(unsafe.Pointer(&b[0]))) % wordSize; n != 0 { + n = wordSize - n + for i := range b[:n] { + b[i] ^= key[pos&3] + pos++ + } + b = b[n:] + } + + // Create aligned word size key. + var k [wordSize]byte + for i := range k { + k[i] = key[(pos+i)&3] + } + kw := *(*uintptr)(unsafe.Pointer(&k)) + + // Mask one word at a time. + n := (len(b) / wordSize) * wordSize + for i := 0; i < n; i += wordSize { + *(*uintptr)(unsafe.Pointer(uintptr(unsafe.Pointer(&b[0])) + uintptr(i))) ^= kw + } + + // Mask one byte at a time for remaining bytes. + b = b[n:] + for i := range b { + b[i] ^= key[pos&3] + pos++ + } + + return pos & 3 +} diff --git a/agent/vendor/github.com/gorilla/websocket/server.go b/agent/vendor/github.com/gorilla/websocket/server.go index c24c4103714..aaedebdbe10 100644 --- a/agent/vendor/github.com/gorilla/websocket/server.go +++ b/agent/vendor/github.com/gorilla/websocket/server.go @@ -21,11 +21,6 @@ type HandshakeError struct { func (e HandshakeError) Error() string { return e.message } -const ( - defaultReadBufferSize = 4096 - defaultWriteBufferSize = 4096 -) - // Upgrader specifies parameters for upgrading an HTTP connection to a // WebSocket connection. type Upgrader struct { @@ -51,6 +46,12 @@ type Upgrader struct { // CheckOrigin is nil, the host in the Origin header must not be set or // must match the host of the request. CheckOrigin func(r *http.Request) bool + + // EnableCompression specify if the server should attempt to negotiate per + // message compression (RFC 7692). Setting this value to true does not + // guarantee that compression will be supported. Currently only "no context + // takeover" modes are supported. + EnableCompression bool } func (u *Upgrader) returnError(w http.ResponseWriter, r *http.Request, status int, reason string) (*Conn, error) { @@ -58,6 +59,7 @@ func (u *Upgrader) returnError(w http.ResponseWriter, r *http.Request, status in if u.Error != nil { u.Error(w, r, status, err) } else { + w.Header().Set("Sec-Websocket-Version", "13") http.Error(w, http.StatusText(status), status) } return nil, err @@ -97,17 +99,28 @@ func (u *Upgrader) selectSubprotocol(r *http.Request, responseHeader http.Header // The responseHeader is included in the response to the client's upgrade // request. Use the responseHeader to specify cookies (Set-Cookie) and the // application negotiated subprotocol (Sec-Websocket-Protocol). +// +// If the upgrade fails, then Upgrade replies to the client with an HTTP error +// response. func (u *Upgrader) Upgrade(w http.ResponseWriter, r *http.Request, responseHeader http.Header) (*Conn, error) { - if values := r.Header["Sec-Websocket-Version"]; len(values) == 0 || values[0] != "13" { + if r.Method != "GET" { + return u.returnError(w, r, http.StatusMethodNotAllowed, "websocket: method not GET") + } + + if _, ok := responseHeader["Sec-Websocket-Extensions"]; ok { + return u.returnError(w, r, http.StatusInternalServerError, "websocket: application specific Sec-Websocket-Extensions headers are unsupported") + } + + if !tokenListContainsValue(r.Header, "Sec-Websocket-Version", "13") { return u.returnError(w, r, http.StatusBadRequest, "websocket: version != 13") } if !tokenListContainsValue(r.Header, "Connection", "upgrade") { - return u.returnError(w, r, http.StatusBadRequest, "websocket: connection header != upgrade") + return u.returnError(w, r, http.StatusBadRequest, "websocket: could not find connection header with token 'upgrade'") } if !tokenListContainsValue(r.Header, "Upgrade", "websocket") { - return u.returnError(w, r, http.StatusBadRequest, "websocket: upgrade != websocket") + return u.returnError(w, r, http.StatusBadRequest, "websocket: could not find upgrade header with token 'websocket'") } checkOrigin := u.CheckOrigin @@ -125,6 +138,18 @@ func (u *Upgrader) Upgrade(w http.ResponseWriter, r *http.Request, responseHeade subprotocol := u.selectSubprotocol(r, responseHeader) + // Negotiate PMCE + var compress bool + if u.EnableCompression { + for _, ext := range parseExtensions(r.Header) { + if ext[""] != "permessage-deflate" { + continue + } + compress = true + break + } + } + var ( netConn net.Conn br *bufio.Reader @@ -147,17 +172,14 @@ func (u *Upgrader) Upgrade(w http.ResponseWriter, r *http.Request, responseHeade return nil, errors.New("websocket: client sent data before handshake is complete") } - readBufSize := u.ReadBufferSize - if readBufSize == 0 { - readBufSize = defaultReadBufferSize - } - writeBufSize := u.WriteBufferSize - if writeBufSize == 0 { - writeBufSize = defaultWriteBufferSize - } - c := newConn(netConn, true, readBufSize, writeBufSize) + c := newConn(netConn, true, u.ReadBufferSize, u.WriteBufferSize) c.subprotocol = subprotocol + if compress { + c.newCompressionWriter = compressNoContextTakeover + c.newDecompressionReader = decompressNoContextTakeover + } + p := c.writeBuf[:0] p = append(p, "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: "...) p = append(p, computeAcceptKey(challengeKey)...) @@ -167,6 +189,9 @@ func (u *Upgrader) Upgrade(w http.ResponseWriter, r *http.Request, responseHeade p = append(p, c.subprotocol...) p = append(p, "\r\n"...) } + if compress { + p = append(p, "Sec-Websocket-Extensions: permessage-deflate; server_no_context_takeover; client_no_context_takeover\r\n"...) + } for k, vs := range responseHeader { if k == "Sec-Websocket-Protocol" { continue @@ -258,3 +283,10 @@ func Subprotocols(r *http.Request) []string { } return protocols } + +// IsWebSocketUpgrade returns true if the client requested upgrade to the +// WebSocket protocol. +func IsWebSocketUpgrade(r *http.Request) bool { + return tokenListContainsValue(r.Header, "Connection", "upgrade") && + tokenListContainsValue(r.Header, "Upgrade", "websocket") +} diff --git a/agent/vendor/github.com/gorilla/websocket/util.go b/agent/vendor/github.com/gorilla/websocket/util.go index ffdc265ed78..9a4908df2ee 100644 --- a/agent/vendor/github.com/gorilla/websocket/util.go +++ b/agent/vendor/github.com/gorilla/websocket/util.go @@ -13,19 +13,6 @@ import ( "strings" ) -// tokenListContainsValue returns true if the 1#token header with the given -// name contains token. -func tokenListContainsValue(header http.Header, name string, value string) bool { - for _, v := range header[name] { - for _, s := range strings.Split(v, ",") { - if strings.EqualFold(value, strings.TrimSpace(s)) { - return true - } - } - } - return false -} - var keyGUID = []byte("258EAFA5-E914-47DA-95CA-C5AB0DC85B11") func computeAcceptKey(challengeKey string) string { @@ -42,3 +29,186 @@ func generateChallengeKey() (string, error) { } return base64.StdEncoding.EncodeToString(p), nil } + +// Octet types from RFC 2616. +var octetTypes [256]byte + +const ( + isTokenOctet = 1 << iota + isSpaceOctet +) + +func init() { + // From RFC 2616 + // + // OCTET = + // CHAR = + // CTL = + // CR = + // LF = + // SP = + // HT = + // <"> = + // CRLF = CR LF + // LWS = [CRLF] 1*( SP | HT ) + // TEXT = + // separators = "(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\" | <"> + // | "/" | "[" | "]" | "?" | "=" | "{" | "}" | SP | HT + // token = 1* + // qdtext = > + + for c := 0; c < 256; c++ { + var t byte + isCtl := c <= 31 || c == 127 + isChar := 0 <= c && c <= 127 + isSeparator := strings.IndexRune(" \t\"(),/:;<=>?@[]\\{}", rune(c)) >= 0 + if strings.IndexRune(" \t\r\n", rune(c)) >= 0 { + t |= isSpaceOctet + } + if isChar && !isCtl && !isSeparator { + t |= isTokenOctet + } + octetTypes[c] = t + } +} + +func skipSpace(s string) (rest string) { + i := 0 + for ; i < len(s); i++ { + if octetTypes[s[i]]&isSpaceOctet == 0 { + break + } + } + return s[i:] +} + +func nextToken(s string) (token, rest string) { + i := 0 + for ; i < len(s); i++ { + if octetTypes[s[i]]&isTokenOctet == 0 { + break + } + } + return s[:i], s[i:] +} + +func nextTokenOrQuoted(s string) (value string, rest string) { + if !strings.HasPrefix(s, "\"") { + return nextToken(s) + } + s = s[1:] + for i := 0; i < len(s); i++ { + switch s[i] { + case '"': + return s[:i], s[i+1:] + case '\\': + p := make([]byte, len(s)-1) + j := copy(p, s[:i]) + escape := true + for i = i + 1; i < len(s); i++ { + b := s[i] + switch { + case escape: + escape = false + p[j] = b + j += 1 + case b == '\\': + escape = true + case b == '"': + return string(p[:j]), s[i+1:] + default: + p[j] = b + j += 1 + } + } + return "", "" + } + } + return "", "" +} + +// tokenListContainsValue returns true if the 1#token header with the given +// name contains token. +func tokenListContainsValue(header http.Header, name string, value string) bool { +headers: + for _, s := range header[name] { + for { + var t string + t, s = nextToken(skipSpace(s)) + if t == "" { + continue headers + } + s = skipSpace(s) + if s != "" && s[0] != ',' { + continue headers + } + if strings.EqualFold(t, value) { + return true + } + if s == "" { + continue headers + } + s = s[1:] + } + } + return false +} + +// parseExtensiosn parses WebSocket extensions from a header. +func parseExtensions(header http.Header) []map[string]string { + + // From RFC 6455: + // + // Sec-WebSocket-Extensions = extension-list + // extension-list = 1#extension + // extension = extension-token *( ";" extension-param ) + // extension-token = registered-token + // registered-token = token + // extension-param = token [ "=" (token | quoted-string) ] + // ;When using the quoted-string syntax variant, the value + // ;after quoted-string unescaping MUST conform to the + // ;'token' ABNF. + + var result []map[string]string +headers: + for _, s := range header["Sec-Websocket-Extensions"] { + for { + var t string + t, s = nextToken(skipSpace(s)) + if t == "" { + continue headers + } + ext := map[string]string{"": t} + for { + s = skipSpace(s) + if !strings.HasPrefix(s, ";") { + break + } + var k string + k, s = nextToken(skipSpace(s[1:])) + if k == "" { + continue headers + } + s = skipSpace(s) + var v string + if strings.HasPrefix(s, "=") { + v, s = nextTokenOrQuoted(skipSpace(s[1:])) + s = skipSpace(s) + } + if s != "" && s[0] != ',' && s[0] != ';' { + continue headers + } + ext[k] = v + } + if s != "" && s[0] != ',' { + continue headers + } + result = append(result, ext) + if s == "" { + continue headers + } + s = s[1:] + } + } + return result +} From e6d30e8219e7e7a7c6835b209571ad80c29181c6 Mon Sep 17 00:00:00 2001 From: Petersen Date: Thu, 15 Dec 2016 10:38:57 -0800 Subject: [PATCH 10/27] Use gorilla/websocket proxy --- agent/acs/client/acs_client.go | 14 +- agent/acs/client/acs_client_test.go | 17 +- agent/acs/client/acs_client_types.go | 31 +-- agent/acs/handler/acs_handler.go | 22 +-- agent/acs/handler/acs_handler_test.go | 76 ++++--- agent/agent.go | 20 +- agent/api/ecsclient/client.go | 8 +- agent/api/ecsclient/client_test.go | 7 +- agent/config/types.go | 5 +- agent/engine/docker_container_engine.go | 4 +- agent/engine/docker_container_engine_test.go | 14 +- agent/engine/engine_integ_test.go | 4 +- agent/stats/common_test.go | 4 +- agent/tcs/client/client.go | 17 +- agent/tcs/client/client_test.go | 9 +- agent/tcs/client/client_types.go | 31 +-- agent/tcs/handler/handler.go | 11 +- agent/tcs/handler/handler_test.go | 43 ++-- agent/wsclient/client.go | 198 +++++++++---------- agent/wsclient/client_test.go | 151 ++++++++++++++ agent/wsclient/mock/client.go | 30 ++- agent/wsclient/mock/utils/utils.go | 11 +- agent/wsclient/types.go | 32 ++- 23 files changed, 465 insertions(+), 294 deletions(-) create mode 100644 agent/wsclient/client_test.go diff --git a/agent/acs/client/acs_client.go b/agent/acs/client/acs_client.go index 56cabe9656f..c46962928b4 100644 --- a/agent/acs/client/acs_client.go +++ b/agent/acs/client/acs_client.go @@ -1,4 +1,4 @@ -// Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the @@ -21,6 +21,7 @@ package acsclient import ( "errors" + "github.com/aws/amazon-ecs-agent/agent/config" "github.com/aws/amazon-ecs-agent/agent/logger" "github.com/aws/amazon-ecs-agent/agent/wsclient" "github.com/aws/aws-sdk-go/aws/credentials" @@ -36,15 +37,14 @@ type clientServer struct { // New returns a client/server to bidirectionally communicate with ACS // The returned struct should have both 'Connect' and 'Serve' called upon it // before being used. -func New(url string, region string, credentialProvider *credentials.Credentials, acceptInvalidCert bool) wsclient.ClientServer { +func New(url string, cfg *config.Config, credentialProvider *credentials.Credentials) wsclient.ClientServer { cs := &clientServer{} cs.URL = url - cs.Region = region cs.CredentialProvider = credentialProvider - cs.AcceptInvalidCert = acceptInvalidCert + cs.AgentConfig = cfg cs.ServiceError = &acsError{} cs.RequestHandlers = make(map[string]wsclient.RequestHandler) - cs.TypeDecoder = &decoder{} + cs.TypeDecoder = NewACSDecoder() return cs } @@ -53,8 +53,8 @@ func New(url string, region string, credentialProvider *credentials.Credentials, // call as unhandled requests will be discarded. func (cs *clientServer) Serve() error { log.Debug("Starting websocket poll loop") - if cs.Conn == nil { - return errors.New("nil connection") + if !cs.IsReady() { + return errors.New("Websocket not ready for connections") } return cs.ConsumeMessages() } diff --git a/agent/acs/client/acs_client_test.go b/agent/acs/client/acs_client_test.go index 32c1527d2ee..beaa6a5680d 100644 --- a/agent/acs/client/acs_client_test.go +++ b/agent/acs/client/acs_client_test.go @@ -1,4 +1,4 @@ -// Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the @@ -23,6 +23,7 @@ import ( "time" "github.com/aws/amazon-ecs-agent/agent/acs/model/ecsacs" + "github.com/aws/amazon-ecs-agent/agent/config" "github.com/aws/amazon-ecs-agent/agent/wsclient" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" @@ -54,6 +55,11 @@ type messageLogger struct { closed bool } +var testCfg = &config.Config{ + AcceptInsecureCert: true, + AWSRegion: "us-east-1", +} + func (ml *messageLogger) WriteMessage(_ int, data []byte) error { if ml.closed { return errors.New("can't write to closed ws") @@ -81,9 +87,10 @@ func (ml *messageLogger) ReadMessage() (int, []byte, error) { func testCS() (wsclient.ClientServer, *messageLogger) { testCreds := credentials.AnonymousCredentials - cs := New("localhost:443", "us-east-1", testCreds, true).(*clientServer) + + cs := New("localhost:443", testCfg, testCreds).(*clientServer) ml := &messageLogger{make([][]byte, 0), make([][]byte, 0), false} - cs.Conn = ml + cs.SetConnection(ml) return cs, ml } @@ -281,7 +288,7 @@ func TestConnect(t *testing.T) { t.Fatal(<-serverErr) }() - cs := New(server.URL, "us-east-1", credentials.AnonymousCredentials, true) + cs := New(server.URL, testCfg, credentials.AnonymousCredentials) // Wait for up to a second for the mock server to launch for i := 0; i < 100; i++ { err = cs.Connect() @@ -352,7 +359,7 @@ func TestConnectClientError(t *testing.T) { })) defer testServer.Close() - cs := New(testServer.URL, "us-east-1", credentials.AnonymousCredentials, true) + cs := New(testServer.URL, testCfg, credentials.AnonymousCredentials) err := cs.Connect() if _, ok := err.(*wsclient.WSError); !ok || err.Error() != "InvalidClusterException: Invalid cluster" { t.Error("Did not get correctly typed error: " + err.Error()) diff --git a/agent/acs/client/acs_client_types.go b/agent/acs/client/acs_client_types.go index 63e7548e2a0..501201140f9 100644 --- a/agent/acs/client/acs_client_types.go +++ b/agent/acs/client/acs_client_types.go @@ -1,4 +1,4 @@ -// Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the @@ -14,12 +14,11 @@ package acsclient import ( - "reflect" - "github.com/aws/amazon-ecs-agent/agent/acs/model/ecsacs" + "github.com/aws/amazon-ecs-agent/agent/wsclient" ) -var acsTypeMappings map[string]reflect.Type +var acsRecognizedTypes []interface{} func init() { // This list is currently *manually updated* and assumes that the generated @@ -29,7 +28,7 @@ func init() { // I couldn't figure out how to get a list of all structs in a package via // reflection, but that would solve this. The alternative is to either parse // the .json model or the generated struct names. - recognizedTypes := []interface{}{ + acsRecognizedTypes = []interface{}{ ecsacs.HeartbeatMessage{}, ecsacs.PayloadMessage{}, ecsacs.CloseMessage{}, @@ -47,26 +46,8 @@ func init() { ecsacs.InactiveInstanceException{}, ecsacs.ErrorMessage{}, } - - acsTypeMappings = make(map[string]reflect.Type) - // This produces a map of: - // "MyMessage": TypeOf(ecsacs.MyMessage) - for _, recognizedType := range recognizedTypes { - acsTypeMappings[reflect.TypeOf(recognizedType).Name()] = reflect.TypeOf(recognizedType) - } -} - -// decoder implments wsclient.TypeDecoder. -type decoder struct{} - -func (dc *decoder) NewOfType(acsType string) (interface{}, bool) { - rtype, ok := acsTypeMappings[acsType] - if !ok { - return nil, false - } - return reflect.New(rtype).Interface(), true } -func (dc *decoder) GetRecognizedTypes() map[string]reflect.Type { - return acsTypeMappings +func NewACSDecoder() wsclient.TypeDecoder { + return wsclient.BuildTypeDecoder(acsRecognizedTypes) } diff --git a/agent/acs/handler/acs_handler.go b/agent/acs/handler/acs_handler.go index 2e1f4738032..5f0a69f292a 100644 --- a/agent/acs/handler/acs_handler.go +++ b/agent/acs/handler/acs_handler.go @@ -1,4 +1,4 @@ -// Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the @@ -79,7 +79,6 @@ type session struct { taskEngine engine.TaskEngine ecsClient api.ECSClient stateManager statemanager.StateManager - acceptInsecureCert bool credentialsManager rolecredentials.Manager ctx context.Context cancel context.CancelFunc @@ -99,7 +98,7 @@ type session struct { // The goal is to make it easier to test and inject dependencies type sessionResources interface { // createACSClient creates a new websocket client - createACSClient(url string) wsclient.ClientServer + createACSClient(url string, cfg *config.Config) wsclient.ClientServer sessionState } @@ -107,9 +106,7 @@ type sessionResources interface { // to create resources needed to connect to ACS and to record session state // for the same type acsSessionResources struct { - region string credentialsProvider *credentials.Credentials - acceptInsecureCert bool // sendCredentials is used to set the 'sendCredentials' URL parameter // used to connect to ACS // It is set to 'true' for the very first successful connection on @@ -131,7 +128,6 @@ type sessionState interface { // NewSession creates a new Session object func NewSession(ctx context.Context, - acceptInsecureCert bool, config *config.Config, deregisterInstanceEventStream *eventstream.EventStream, containerInstanceArn string, @@ -140,13 +136,12 @@ func NewSession(ctx context.Context, stateManager statemanager.StateManager, taskEngine engine.TaskEngine, credentialsManager rolecredentials.Manager) Session { - resources := newSessionResources(config.AWSRegion, credentialsProvider, acceptInsecureCert) + resources := newSessionResources(credentialsProvider) backoff := utils.NewSimpleBackoff(connectionBackoffMin, connectionBackoffMax, connectionBackoffJitter, connectionBackoffMultiplier) derivedContext, cancel := context.WithCancel(ctx) return &session{ - acceptInsecureCert: acceptInsecureCert, agentConfig: config, deregisterInstanceEventStream: deregisterInstanceEventStream, containerInstanceARN: containerInstanceArn, @@ -239,7 +234,7 @@ func (acsSession *session) startSessionOnce() error { } url := acsWsURL(acsEndpoint, acsSession.agentConfig.Cluster, acsSession.containerInstanceARN, acsSession.taskEngine, acsSession.resources) - client := acsSession.resources.createACSClient(url) + client := acsSession.resources.createACSClient(url, acsSession.agentConfig) defer client.Close() // Start inactivity timer for closing the connection @@ -347,9 +342,8 @@ func (acsSession *session) heartbeatJitter() time.Duration { } // createACSClient creates the ACS Client using the specified URL -func (acsResources *acsSessionResources) createACSClient(url string) wsclient.ClientServer { - return acsclient.New( - url, acsResources.region, acsResources.credentialsProvider, acsResources.acceptInsecureCert) +func (acsResources *acsSessionResources) createACSClient(url string, cfg *config.Config) wsclient.ClientServer { + return acsclient.New(url, cfg, acsResources.credentialsProvider) } // connectedToACS records a successful connection to ACS @@ -364,11 +358,9 @@ func (acsResources *acsSessionResources) getSendCredentialsURLParameter() string return strconv.FormatBool(acsResources.sendCredentials) } -func newSessionResources(region string, credentialsProvider *credentials.Credentials, acceptInsecureCert bool) sessionResources { +func newSessionResources(credentialsProvider *credentials.Credentials) sessionResources { return &acsSessionResources{ - region: region, credentialsProvider: credentialsProvider, - acceptInsecureCert: acceptInsecureCert, sendCredentials: true, } } diff --git a/agent/acs/handler/acs_handler_test.go b/agent/acs/handler/acs_handler_test.go index 58b7b08488b..12726677f2b 100644 --- a/agent/acs/handler/acs_handler_test.go +++ b/agent/acs/handler/acs_handler_test.go @@ -113,11 +113,16 @@ const ( acsURL = "http://endpoint.tld" ) +var testConfig = &config.Config{ + Cluster: "someCluster", + AcceptInsecureCert: true, +} + type mockSessionResources struct { client wsclient.ClientServer } -func (m *mockSessionResources) createACSClient(url string) wsclient.ClientServer { +func (m *mockSessionResources) createACSClient(url string, cfg *config.Config) wsclient.ClientServer { return m.client } @@ -201,11 +206,10 @@ func TestHandlerReconnectsOnConnectErrors(t *testing.T) { acsSession := session{ containerInstanceARN: "myArn", credentialsProvider: credentials.AnonymousCredentials, - agentConfig: &config.Config{Cluster: "someCluster"}, + agentConfig: testConfig, taskEngine: taskEngine, ecsClient: ecsClient, stateManager: statemanager, - acceptInsecureCert: true, backoff: utils.NewSimpleBackoff(connectionBackoffMin, connectionBackoffMax, connectionBackoffJitter, connectionBackoffMultiplier), ctx: ctx, cancel: cancel, @@ -344,12 +348,11 @@ func TestHandlerReconnectsWithoutBackoffOnEOFError(t *testing.T) { acsSession := session{ containerInstanceARN: "myArn", credentialsProvider: credentials.AnonymousCredentials, - agentConfig: &config.Config{Cluster: "someCluster"}, + agentConfig: testConfig, taskEngine: taskEngine, ecsClient: ecsClient, deregisterInstanceEventStream: deregisterInstanceEventStream, stateManager: statemanager, - acceptInsecureCert: true, backoff: mockBackoff, ctx: ctx, cancel: cancel, @@ -406,12 +409,11 @@ func TestHandlerReconnectsWithBackoffOnNonEOFError(t *testing.T) { acsSession := session{ containerInstanceARN: "myArn", credentialsProvider: credentials.AnonymousCredentials, - agentConfig: &config.Config{Cluster: "someCluster"}, + agentConfig: testConfig, taskEngine: taskEngine, ecsClient: ecsClient, deregisterInstanceEventStream: deregisterInstanceEventStream, stateManager: statemanager, - acceptInsecureCert: true, backoff: mockBackoff, ctx: ctx, cancel: cancel, @@ -464,12 +466,11 @@ func TestHandlerGeneratesDeregisteredInstanceEvent(t *testing.T) { acsSession := session{ containerInstanceARN: "myArn", credentialsProvider: credentials.AnonymousCredentials, - agentConfig: &config.Config{Cluster: "someCluster"}, + agentConfig: testConfig, taskEngine: taskEngine, ecsClient: ecsClient, deregisterInstanceEventStream: deregisterInstanceEventStream, stateManager: statemanager, - acceptInsecureCert: true, backoff: utils.NewSimpleBackoff(connectionBackoffMin, connectionBackoffMax, connectionBackoffJitter, connectionBackoffMultiplier), ctx: ctx, cancel: cancel, @@ -532,12 +533,11 @@ func TestHandlerReconnectDelayForInactiveInstanceError(t *testing.T) { acsSession := session{ containerInstanceARN: "myArn", credentialsProvider: credentials.AnonymousCredentials, - agentConfig: &config.Config{Cluster: "someCluster"}, + agentConfig: testConfig, taskEngine: taskEngine, ecsClient: ecsClient, deregisterInstanceEventStream: deregisterInstanceEventStream, stateManager: statemanager, - acceptInsecureCert: true, backoff: utils.NewSimpleBackoff(connectionBackoffMin, connectionBackoffMax, connectionBackoffJitter, connectionBackoffMultiplier), ctx: ctx, cancel: cancel, @@ -589,11 +589,10 @@ func TestHandlerReconnectsOnServeErrors(t *testing.T) { acsSession := session{ containerInstanceARN: "myArn", credentialsProvider: credentials.AnonymousCredentials, - agentConfig: &config.Config{Cluster: "someCluster"}, + agentConfig: testConfig, taskEngine: taskEngine, ecsClient: ecsClient, stateManager: statemanager, - acceptInsecureCert: true, backoff: utils.NewSimpleBackoff(connectionBackoffMin, connectionBackoffMax, connectionBackoffJitter, connectionBackoffMultiplier), ctx: ctx, cancel: cancel, @@ -639,11 +638,10 @@ func TestHandlerStopsWhenContextIsCancelled(t *testing.T) { acsSession := session{ containerInstanceARN: "myArn", credentialsProvider: credentials.AnonymousCredentials, - agentConfig: &config.Config{Cluster: "someCluster"}, + agentConfig: testConfig, taskEngine: taskEngine, ecsClient: ecsClient, stateManager: statemanager, - acceptInsecureCert: true, backoff: utils.NewSimpleBackoff(connectionBackoffMin, connectionBackoffMax, connectionBackoffJitter, connectionBackoffMultiplier), ctx: ctx, cancel: cancel, @@ -693,11 +691,10 @@ func TestHandlerReconnectsOnDiscoverPollEndpointError(t *testing.T) { acsSession := session{ containerInstanceARN: "myArn", credentialsProvider: credentials.AnonymousCredentials, - agentConfig: &config.Config{Cluster: "someCluster"}, + agentConfig: testConfig, taskEngine: taskEngine, ecsClient: ecsClient, stateManager: statemanager, - acceptInsecureCert: true, backoff: utils.NewSimpleBackoff(connectionBackoffMin, connectionBackoffMax, connectionBackoffJitter, connectionBackoffMultiplier), ctx: ctx, cancel: cancel, @@ -759,11 +756,10 @@ func TestConnectionIsClosedOnIdle(t *testing.T) { acsSession := session{ containerInstanceARN: "myArn", credentialsProvider: credentials.AnonymousCredentials, - agentConfig: &config.Config{Cluster: "someCluster"}, + agentConfig: testConfig, taskEngine: taskEngine, ecsClient: ecsClient, stateManager: statemanager, - acceptInsecureCert: true, ctx: context.Background(), backoff: utils.NewSimpleBackoff(connectionBackoffMin, connectionBackoffMax, connectionBackoffJitter, connectionBackoffMultiplier), resources: &mockSessionResources{}, @@ -781,7 +777,7 @@ func TestConnectionIsClosedOnIdle(t *testing.T) { <-connectionClosed } -func TestHandlerDoesntLeakGouroutines(t *testing.T) { +func TestHandlerDoesntLeakGoroutines(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() taskEngine := engine.NewMockTaskEngine(ctrl) @@ -815,17 +811,14 @@ func TestHandlerDoesntLeakGouroutines(t *testing.T) { acsSession := session{ containerInstanceARN: "myArn", credentialsProvider: credentials.AnonymousCredentials, - agentConfig: &config.Config{Cluster: "someCluster"}, + agentConfig: testConfig, taskEngine: taskEngine, ecsClient: ecsClient, stateManager: statemanager, - acceptInsecureCert: true, ctx: ctx, backoff: utils.NewSimpleBackoff(connectionBackoffMin, connectionBackoffMax, connectionBackoffJitter, connectionBackoffMultiplier), - resources: newSessionResources("", credentials.AnonymousCredentials, true), + resources: newSessionResources(credentials.AnonymousCredentials), credentialsManager: rolecredentials.NewManager(), - _heartbeatTimeout: 20 * time.Millisecond, - _heartbeatJitter: 10 * time.Millisecond, } acsSession.Start() ended <- true @@ -844,9 +837,12 @@ func TestHandlerDoesntLeakGouroutines(t *testing.T) { cancel() <-ended + // The number of goroutines finishing in the MockACSServer will affect + // the result unless we wait here. + time.Sleep(10 * time.Millisecond) afterGoroutines := runtime.NumGoroutine() - t.Logf("Gorutines after 1 and after 100 acs messages: %v and %v", beforeGoroutines, afterGoroutines) + t.Logf("Goroutines after 1 and after %v acs messages: %v and %v", timesConnected, beforeGoroutines, afterGoroutines) if timesConnected < 50 { t.Fatal("Expected times connected to be a large number, was ", timesConnected) @@ -855,6 +851,7 @@ func TestHandlerDoesntLeakGouroutines(t *testing.T) { t.Error("Goroutine leak, oh no!") pprof.Lookup("goroutine").WriteTo(os.Stdout, 1) } + } // TestStartSessionHandlesRefreshCredentialsMessages tests the agent restart @@ -893,8 +890,7 @@ func TestStartSessionHandlesRefreshCredentialsMessages(t *testing.T) { ended := make(chan bool, 1) go func() { acsSession := NewSession(ctx, - true, - &config.Config{Cluster: "someCluster"}, + testConfig, nil, "myArn", credentials.AnonymousCredentials, @@ -960,7 +956,7 @@ func TestStartSessionHandlesRefreshCredentialsMessages(t *testing.T) { // TestACSSessionResourcesCorrectlySetsSendCredentials tests if acsSessionResources // struct correctly sets 'sendCredentials' func TestACSSessionResourcesCorrectlySetsSendCredentials(t *testing.T) { - acsResources := newSessionResources("", nil, true) + acsResources := newSessionResources(nil) // Validate that 'sendCredentials' is set to true on create sendCredentials := acsResources.getSendCredentialsURLParameter() if sendCredentials != "true" { @@ -992,7 +988,7 @@ func TestHandlerReconnectsCorrectlySetsSendCredentialsURLParameter(t *testing.T) mockWsClient.EXPECT().AddRequestHandler(gomock.Any()).AnyTimes() mockWsClient.EXPECT().Close().Return(nil).AnyTimes() mockWsClient.EXPECT().Serve().Return(io.EOF).AnyTimes() - resources := newSessionResources("", credentials.AnonymousCredentials, true) + resources := newSessionResources(credentials.AnonymousCredentials) gomock.InOrder( // When the websocket client connects to ACS for the first // time, 'sendCredentials' should be set to true @@ -1009,11 +1005,10 @@ func TestHandlerReconnectsCorrectlySetsSendCredentialsURLParameter(t *testing.T) acsSession := session{ containerInstanceARN: "myArn", credentialsProvider: credentials.AnonymousCredentials, - agentConfig: &config.Config{Cluster: "someCluster"}, + agentConfig: testConfig, taskEngine: taskEngine, ecsClient: ecsClient, stateManager: statemanager, - acceptInsecureCert: true, ctx: ctx, resources: resources, backoff: utils.NewSimpleBackoff(connectionBackoffMin, connectionBackoffMax, connectionBackoffJitter, connectionBackoffMultiplier), @@ -1040,20 +1035,14 @@ func startMockAcsServer(t *testing.T, closeWS <-chan bool) (*httptest.Server, ch requestsChan := make(chan string, 1) errChan := make(chan error, 1) - serverRestart := make(chan bool, 1) upgrader := websocket.Upgrader{ReadBufferSize: 1024, WriteBufferSize: 1024} handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ws, err := upgrader.Upgrade(w, r, nil) - go func() { - <-closeWS - ws.WriteMessage(websocket.CloseMessage, nil) - ws.Close() - serverRestart <- true - errChan <- io.EOF - }() + if err != nil { errChan <- err } + go func() { _, msg, err := ws.ReadMessage() if err != nil { @@ -1069,10 +1058,15 @@ func startMockAcsServer(t *testing.T, closeWS <-chan bool) (*httptest.Server, ch if err != nil { errChan <- err } - case <-serverRestart: + + case <-closeWS: + ws.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")) + ws.Close() + errChan <- io.EOF // Quit listening to serverChan if we've been closed return } + } }) diff --git a/agent/agent.go b/agent/agent.go index 92608a512e6..13b76caad6a 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -1,4 +1,4 @@ -// Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the @@ -33,7 +33,6 @@ import ( "github.com/aws/amazon-ecs-agent/agent/eventstream" "github.com/aws/amazon-ecs-agent/agent/handlers" credentialshandler "github.com/aws/amazon-ecs-agent/agent/handlers/credentials" - "github.com/aws/amazon-ecs-agent/agent/httpclient" "github.com/aws/amazon-ecs-agent/agent/logger" "github.com/aws/amazon-ecs-agent/agent/sighandlers" "github.com/aws/amazon-ecs-agent/agent/sighandlers/exitcodes" @@ -41,6 +40,7 @@ import ( "github.com/aws/amazon-ecs-agent/agent/tcs/handler" "github.com/aws/amazon-ecs-agent/agent/utils" "github.com/aws/amazon-ecs-agent/agent/version" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/defaults" log "github.com/cihub/seelog" @@ -92,14 +92,15 @@ func _main() int { } log.Infof("Starting Agent: %s", version.String()) - if *acceptInsecureCert { + if aws.BoolValue(acceptInsecureCert) { log.Warn("SSL certificate verification disabled. This is not recommended.") } log.Info("Loading configuration") cfg, cfgErr := config.NewConfig(ec2MetadataClient) + cfg.AcceptInsecureCert = aws.BoolValue(acceptInsecureCert) // Load cfg and create Docker client before doing 'versionFlag' so that it has the DOCKER_HOST variable loaded if needed clientFactory := dockerclient.NewFactory(cfg.DockerEndpoint) - dockerClient, err := engine.NewDockerGoClient(clientFactory, *acceptInsecureCert, cfg) + dockerClient, err := engine.NewDockerGoClient(clientFactory, cfg) if err != nil { log.Criticalf("Error creating Docker client: %v", err) return exitcodes.ExitError @@ -204,8 +205,7 @@ func _main() int { if preflightCreds, err := credentialProvider.Get(); err != nil || preflightCreds.AccessKeyID == "" { log.Warnf("Error getting valid credentials (AKID %s): %v", preflightCreds.AccessKeyID, err) } - client := ecsclient.NewECSClient(credentialProvider, cfg, - httpclient.New(ecsclient.RoundtripTimeout, *acceptInsecureCert), ec2MetadataClient) + client := ecsclient.NewECSClient(credentialProvider, cfg, ec2MetadataClient) if containerInstanceArn == "" { log.Info("Registering Instance with ECS") @@ -266,13 +266,12 @@ func _main() int { deregisterInstanceEventStream.StartListening() telemetrySessionParams := tcshandler.TelemetrySessionParams{ - ContainerInstanceArn: containerInstanceArn, - CredentialProvider: credentialProvider, - Cfg: cfg, + CredentialProvider: credentialProvider, + Cfg: cfg, + ContainerInstanceArn: containerInstanceArn, DeregisterInstanceEventStream: deregisterInstanceEventStream, ContainerChangeEventStream: containerChangeEventStream, DockerClient: dockerClient, - AcceptInvalidCert: *acceptInsecureCert, ECSClient: client, TaskEngine: taskEngine, } @@ -282,7 +281,6 @@ func _main() int { acsSession := acshandler.NewSession( ctx, - *acceptInsecureCert, cfg, deregisterInstanceEventStream, containerInstanceArn, diff --git a/agent/api/ecsclient/client.go b/agent/api/ecsclient/client.go index 3a54cc44d83..7cb3b78a917 100644 --- a/agent/api/ecsclient/client.go +++ b/agent/api/ecsclient/client.go @@ -1,4 +1,4 @@ -// Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the @@ -15,7 +15,6 @@ package ecsclient import ( "errors" - "net/http" "runtime" "strings" "time" @@ -25,6 +24,7 @@ import ( "github.com/aws/amazon-ecs-agent/agent/config" "github.com/aws/amazon-ecs-agent/agent/ec2" "github.com/aws/amazon-ecs-agent/agent/ecs_client/model/ecs" + "github.com/aws/amazon-ecs-agent/agent/httpclient" "github.com/aws/amazon-ecs-agent/agent/logger" "github.com/aws/amazon-ecs-agent/agent/utils" "github.com/aws/aws-sdk-go/aws" @@ -53,11 +53,11 @@ type APIECSClient struct { pollEndpoinCache async.Cache } -func NewECSClient(credentialProvider *credentials.Credentials, config *config.Config, httpClient *http.Client, ec2MetadataClient ec2.EC2MetadataClient) api.ECSClient { +func NewECSClient(credentialProvider *credentials.Credentials, config *config.Config, ec2MetadataClient ec2.EC2MetadataClient) api.ECSClient { var ecsConfig aws.Config ecsConfig.Credentials = credentialProvider ecsConfig.Region = &config.AWSRegion - ecsConfig.HTTPClient = httpClient + ecsConfig.HTTPClient = httpclient.New(RoundtripTimeout, config.AcceptInsecureCert) if config.APIEndpoint != "" { ecsConfig.Endpoint = &config.APIEndpoint } diff --git a/agent/api/ecsclient/client_test.go b/agent/api/ecsclient/client_test.go index af845f215ab..725cff112a1 100644 --- a/agent/api/ecsclient/client_test.go +++ b/agent/api/ecsclient/client_test.go @@ -1,4 +1,4 @@ -// Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the @@ -16,7 +16,6 @@ package ecsclient import ( "errors" "fmt" - "net/http" "reflect" "strings" "testing" @@ -45,7 +44,7 @@ func NewMockClient(ctrl *gomock.Controller, ec2Metadata ec2.EC2MetadataClient, a &config.Config{Cluster: configuredCluster, AWSRegion: "us-east-1", InstanceAttributes: additionalAttributes, - }, http.DefaultClient, ec2Metadata) + }, ec2Metadata) mockSDK := mock_api.NewMockECSSDK(ctrl) mockSubmitStateSDK := mock_api.NewMockECSSubmitStateSDK(ctrl) client.(*APIECSClient).SetSDK(mockSDK) @@ -382,7 +381,7 @@ func TestRegisterBlankCluster(t *testing.T) { defer mockCtrl.Finish() mockEC2Metadata := mock_ec2.NewMockEC2MetadataClient(mockCtrl) // Test the special 'empty cluster' behavior of creating 'default' - client := NewECSClient(credentials.AnonymousCredentials, &config.Config{Cluster: "", AWSRegion: "us-east-1"}, http.DefaultClient, mockEC2Metadata) + client := NewECSClient(credentials.AnonymousCredentials, &config.Config{Cluster: "", AWSRegion: "us-east-1"}, mockEC2Metadata) mc := mock_api.NewMockECSSDK(mockCtrl) client.(*APIECSClient).SetSDK(mc) diff --git a/agent/config/types.go b/agent/config/types.go index b853c273cf5..146e048ac77 100644 --- a/agent/config/types.go +++ b/agent/config/types.go @@ -1,4 +1,4 @@ -// Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the @@ -140,6 +140,9 @@ type Config struct { // ECS service and used to influence behavior such as launch // placement. InstanceAttributes map[string]string + + // Set if clients validate ssl certificates. Used mainly for testing + AcceptInsecureCert bool `json:"-"` } // SensitiveRawMessage is a struct to store some data that should not be logged diff --git a/agent/engine/docker_container_engine.go b/agent/engine/docker_container_engine.go index adcc1136fa1..58acf2d1a52 100644 --- a/agent/engine/docker_container_engine.go +++ b/agent/engine/docker_container_engine.go @@ -131,7 +131,7 @@ func (dg *dockerGoClient) WithVersion(version dockerclient.DockerVersion) Docker var scratchCreateLock sync.Mutex // NewDockerGoClient creates a new DockerGoClient -func NewDockerGoClient(clientFactory dockerclient.Factory, acceptInsecureCert bool, cfg *config.Config) (DockerClient, error) { +func NewDockerGoClient(clientFactory dockerclient.Factory, cfg *config.Config) (DockerClient, error) { client, err := clientFactory.GetDefaultClient() if err != nil { log.Error("Unable to connect to docker daemon. Ensure docker is running.", "err", err) @@ -149,7 +149,7 @@ func NewDockerGoClient(clientFactory dockerclient.Factory, acceptInsecureCert bo return &dockerGoClient{ clientFactory: clientFactory, auth: dockerauth.NewDockerAuthProvider(cfg.EngineAuthType, cfg.EngineAuthData.Contents()), - ecrClientFactory: ecr.NewECRFactory(acceptInsecureCert), + ecrClientFactory: ecr.NewECRFactory(cfg.AcceptInsecureCert), config: cfg, }, nil } diff --git a/agent/engine/docker_container_engine_test.go b/agent/engine/docker_container_engine_test.go index e3696156e1d..5d9b87c3db5 100644 --- a/agent/engine/docker_container_engine_test.go +++ b/agent/engine/docker_container_engine_test.go @@ -1,5 +1,5 @@ // +build !integration -// Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the @@ -65,7 +65,7 @@ func dockerClientSetupWithConfig(t *testing.T, conf config.Config) (*mock_docker mockTime := mock_ttime.NewMockTime(ctrl) conf.EngineAuthData = config.NewSensitiveRawMessage([]byte{}) - client, _ := NewDockerGoClient(factory, false, &conf) + client, _ := NewDockerGoClient(factory, &conf) goClient, _ := client.(*dockerGoClient) ecrClientFactory := mock_ecr.NewMockECRFactory(ctrl) goClient.ecrClientFactory = ecrClientFactory @@ -241,7 +241,7 @@ func TestPullImageECRSuccess(t *testing.T) { mockDocker.EXPECT().Ping().AnyTimes().Return(nil) factory := mock_dockerclient.NewMockFactory(ctrl) factory.EXPECT().GetDefaultClient().AnyTimes().Return(mockDocker, nil) - client, _ := NewDockerGoClient(factory, false, defaultTestConfig()) + client, _ := NewDockerGoClient(factory, defaultTestConfig()) goClient, _ := client.(*dockerGoClient) ecrClientFactory := mock_ecr.NewMockECRFactory(ctrl) ecrClient := mock_ecr.NewMockECRClient(ctrl) @@ -297,7 +297,7 @@ func TestPullImageECRAuthFail(t *testing.T) { mockDocker.EXPECT().Ping().AnyTimes().Return(nil) factory := mock_dockerclient.NewMockFactory(ctrl) factory.EXPECT().GetDefaultClient().AnyTimes().Return(mockDocker, nil) - client, _ := NewDockerGoClient(factory, false, defaultTestConfig()) + client, _ := NewDockerGoClient(factory, defaultTestConfig()) goClient, _ := client.(*dockerGoClient) ecrClientFactory := mock_ecr.NewMockECRFactory(ctrl) ecrClient := mock_ecr.NewMockECRClient(ctrl) @@ -730,7 +730,7 @@ func TestPingFailError(t *testing.T) { mockDocker.EXPECT().Ping().Return(errors.New("err")) factory := mock_dockerclient.NewMockFactory(ctrl) factory.EXPECT().GetDefaultClient().Return(mockDocker, nil) - _, err := NewDockerGoClient(factory, false, defaultTestConfig()) + _, err := NewDockerGoClient(factory, defaultTestConfig()) if err == nil { t.Fatal("Expected ping error to result in constructor fail") } @@ -743,7 +743,7 @@ func TestUsesVersionedClient(t *testing.T) { mockDocker.EXPECT().Ping().Return(nil) factory := mock_dockerclient.NewMockFactory(ctrl) factory.EXPECT().GetDefaultClient().Return(mockDocker, nil) - client, err := NewDockerGoClient(factory, false, defaultTestConfig()) + client, err := NewDockerGoClient(factory, defaultTestConfig()) if err != nil { t.Fatal(err) } @@ -764,7 +764,7 @@ func TestUnavailableVersionError(t *testing.T) { mockDocker.EXPECT().Ping().Return(nil) factory := mock_dockerclient.NewMockFactory(ctrl) factory.EXPECT().GetDefaultClient().Return(mockDocker, nil) - client, err := NewDockerGoClient(factory, false, defaultTestConfig()) + client, err := NewDockerGoClient(factory, defaultTestConfig()) if err != nil { t.Fatal(err) } diff --git a/agent/engine/engine_integ_test.go b/agent/engine/engine_integ_test.go index d91edc42234..e81158d66b8 100644 --- a/agent/engine/engine_integ_test.go +++ b/agent/engine/engine_integ_test.go @@ -1,5 +1,5 @@ // +build integration -// Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the @@ -68,7 +68,7 @@ func setup(cfg *config.Config, t *testing.T) (TaskEngine, func(), credentials.Ma t.Skip("Docker not running") } clientFactory := dockerclient.NewFactory(dockerEndpoint) - dockerClient, err := NewDockerGoClient(clientFactory, false, cfg) + dockerClient, err := NewDockerGoClient(clientFactory, cfg) if err != nil { t.Fatalf("Error creating Docker client: %v", err) } diff --git a/agent/stats/common_test.go b/agent/stats/common_test.go index ffe3be5f13d..7045065109a 100644 --- a/agent/stats/common_test.go +++ b/agent/stats/common_test.go @@ -1,4 +1,4 @@ -// Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the @@ -62,7 +62,7 @@ var defaultContainerInstance = "ci" func init() { cfg.EngineAuthData = config.NewSensitiveRawMessage([]byte{}) - dockerClient, _ = ecsengine.NewDockerGoClient(clientFactory, false, &cfg) + dockerClient, _ = ecsengine.NewDockerGoClient(clientFactory, &cfg) } // eventStream returns the event stream used to receive container change events diff --git a/agent/tcs/client/client.go b/agent/tcs/client/client.go index c869fb121a7..b0092bc3ce1 100644 --- a/agent/tcs/client/client.go +++ b/agent/tcs/client/client.go @@ -20,6 +20,7 @@ import ( "net/http" "time" + "github.com/aws/amazon-ecs-agent/agent/config" "github.com/aws/amazon-ecs-agent/agent/stats" "github.com/aws/amazon-ecs-agent/agent/tcs/model/ecstcs" "github.com/aws/amazon-ecs-agent/agent/utils" @@ -27,7 +28,6 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/cihub/seelog" - "github.com/gorilla/websocket" ) // tasksInMessage is the maximum number of tasks that can be sent in a message to the backend @@ -46,19 +46,18 @@ type clientServer struct { // New returns a client/server to bidirectionally communicate with the backend. // The returned struct should have both 'Connect' and 'Serve' called upon it // before being used. -func New(url string, region string, credentialProvider *credentials.Credentials, acceptInvalidCert bool, statsEngine stats.Engine, publishMetricsInterval time.Duration) wsclient.ClientServer { +func New(url string, cfg *config.Config, credentialProvider *credentials.Credentials, statsEngine stats.Engine, publishMetricsInterval time.Duration) wsclient.ClientServer { cs := &clientServer{ statsEngine: statsEngine, publishTicker: nil, publishMetricsInterval: publishMetricsInterval, } cs.URL = url - cs.Region = region + cs.AgentConfig = cfg cs.CredentialProvider = credentialProvider - cs.AcceptInvalidCert = acceptInvalidCert cs.ServiceError = &tcsError{} cs.RequestHandlers = make(map[string]wsclient.RequestHandler) - cs.TypeDecoder = &TcsDecoder{} + cs.TypeDecoder = NewTCSDecoder() return cs } @@ -67,8 +66,8 @@ func New(url string, region string, credentialProvider *credentials.Credentials, // call as unhandled requests will be discarded. func (cs *clientServer) Serve() error { seelog.Debug("TCS client starting websocket poll loop") - if cs.Conn == nil { - return fmt.Errorf("nil connection") + if !cs.IsReady() { + return fmt.Errorf("Websocket not ready for connections") } if cs.statsEngine == nil { @@ -96,7 +95,7 @@ func (cs *clientServer) MakeRequest(input interface{}) error { // Over the wire we send something like // {"type":"AckRequest","message":{"messageId":"xyz"}} - return cs.Conn.WriteMessage(websocket.TextMessage, data) + return cs.WriteMessage(data) } func (cs *clientServer) signRequest(payload []byte) []byte { @@ -104,7 +103,7 @@ func (cs *clientServer) signRequest(payload []byte) []byte { // NewRequest never returns an error if the url parses and we just verified // it did above request, _ := http.NewRequest("GET", cs.URL, reqBody) - utils.SignHTTPRequest(request, cs.Region, "ecs", cs.CredentialProvider, aws.ReadSeekCloser(reqBody)) + utils.SignHTTPRequest(request, cs.AgentConfig.AWSRegion, "ecs", cs.CredentialProvider, aws.ReadSeekCloser(reqBody)) request.Header.Add("Host", request.Host) var dataBuffer bytes.Buffer diff --git a/agent/tcs/client/client_test.go b/agent/tcs/client/client_test.go index 7d1639c450a..f4f4943aa3c 100644 --- a/agent/tcs/client/client_test.go +++ b/agent/tcs/client/client_test.go @@ -26,6 +26,7 @@ import ( "testing" "time" + "github.com/aws/amazon-ecs-agent/agent/config" "github.com/aws/amazon-ecs-agent/agent/eventstream" "github.com/aws/amazon-ecs-agent/agent/tcs/model/ecstcs" "github.com/aws/amazon-ecs-agent/agent/wsclient" @@ -222,9 +223,13 @@ func TestPublishOnceNonIdleStatsEngine(t *testing.T) { func testCS() (wsclient.ClientServer, *messageLogger) { testCreds := credentials.AnonymousCredentials - cs := New("localhost:443", "us-east-1", testCreds, true, &mockStatsEngine{}, testPublishMetricsInterval).(*clientServer) + cfg := &config.Config{ + AWSRegion: "us-east-1", + AcceptInsecureCert: true, + } + cs := New("localhost:443", cfg, testCreds, &mockStatsEngine{}, testPublishMetricsInterval).(*clientServer) ml := &messageLogger{make([][]byte, 0), make([][]byte, 0), false} - cs.Conn = ml + cs.SetConnection(ml) return cs, ml } diff --git a/agent/tcs/client/client_types.go b/agent/tcs/client/client_types.go index f1a117d3b31..d780e1b6e68 100644 --- a/agent/tcs/client/client_types.go +++ b/agent/tcs/client/client_types.go @@ -1,4 +1,4 @@ -// Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the @@ -20,12 +20,11 @@ package tcsclient import ( - "reflect" - "github.com/aws/amazon-ecs-agent/agent/tcs/model/ecstcs" + "github.com/aws/amazon-ecs-agent/agent/wsclient" ) -var tcsTypeMappings map[string]reflect.Type +var tcsTypes []interface{} func init() { // This list is currently *manually updated* and assumes that the generated @@ -35,7 +34,7 @@ func init() { // I couldn't figure out how to get a list of all structs in a package via // reflection, but that would solve this. The alternative is to either parse // the .json model or the generated struct names. - recognizedTypes := []interface{}{ + tcsTypes = []interface{}{ ecstcs.StopTelemetrySessionMessage{}, ecstcs.AckPublishMetric{}, ecstcs.HeartbeatMessage{}, @@ -46,26 +45,8 @@ func init() { ecstcs.ResourceValidationException{}, ecstcs.InvalidParameterException{}, } - - tcsTypeMappings = make(map[string]reflect.Type) - // This produces a map of: - // "MyMessage": TypeOf(ecstcs.MyMessage) - for _, recognizedType := range recognizedTypes { - tcsTypeMappings[reflect.TypeOf(recognizedType).Name()] = reflect.TypeOf(recognizedType) - } -} - -// TcsDecoder implments wsclient.TypeDecoder. -type TcsDecoder struct{} - -func (dc *TcsDecoder) NewOfType(tcsType string) (interface{}, bool) { - rtype, ok := tcsTypeMappings[tcsType] - if !ok { - return nil, false - } - return reflect.New(rtype).Interface(), true } -func (dc *TcsDecoder) GetRecognizedTypes() map[string]reflect.Type { - return tcsTypeMappings +func NewTCSDecoder() wsclient.TypeDecoder { + return wsclient.BuildTypeDecoder(tcsTypes) } diff --git a/agent/tcs/handler/handler.go b/agent/tcs/handler/handler.go index 286f5332d59..2dfb530298a 100644 --- a/agent/tcs/handler/handler.go +++ b/agent/tcs/handler/handler.go @@ -1,4 +1,4 @@ -// Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the @@ -19,6 +19,7 @@ import ( "strings" "time" + "github.com/aws/amazon-ecs-agent/agent/config" "github.com/aws/amazon-ecs-agent/agent/eventstream" "github.com/aws/amazon-ecs-agent/agent/stats" "github.com/aws/amazon-ecs-agent/agent/tcs/client" @@ -90,11 +91,13 @@ func startTelemetrySession(params TelemetrySessionParams, statsEngine stats.Engi } log.Debugf("Connecting to TCS endpoint %v", tcsEndpoint) url := formatURL(tcsEndpoint, params.Cfg.Cluster, params.ContainerInstanceArn) - return startSession(url, params.Cfg.AWSRegion, params.CredentialProvider, params.AcceptInvalidCert, statsEngine, defaultHeartbeatTimeout, defaultHeartbeatJitter, defaultPublishMetricsInterval, params.DeregisterInstanceEventStream) + return startSession(url, params.Cfg, params.CredentialProvider, statsEngine, defaultHeartbeatTimeout, defaultHeartbeatJitter, defaultPublishMetricsInterval, params.DeregisterInstanceEventStream) } -func startSession(url string, region string, credentialProvider *credentials.Credentials, acceptInvalidCert bool, statsEngine stats.Engine, heartbeatTimeout, heartbeatJitter, publishMetricsInterval time.Duration, deregisterInstanceEventStream *eventstream.EventStream) error { - client := tcsclient.New(url, region, credentialProvider, acceptInvalidCert, statsEngine, publishMetricsInterval) +func startSession(url string, cfg *config.Config, credentialProvider *credentials.Credentials, + statsEngine stats.Engine, heartbeatTimeout, heartbeatJitter, publishMetricsInterval time.Duration, + deregisterInstanceEventStream *eventstream.EventStream) error { + client := tcsclient.New(url, cfg, credentialProvider, statsEngine, publishMetricsInterval) defer client.Close() err := deregisterInstanceEventStream.Subscribe(deregisterContainerInstanceHandler, client.Disconnect) diff --git a/agent/tcs/handler/handler_test.go b/agent/tcs/handler/handler_test.go index 10c47cec775..5560e0d6bae 100644 --- a/agent/tcs/handler/handler_test.go +++ b/agent/tcs/handler/handler_test.go @@ -1,4 +1,4 @@ -// Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the @@ -24,6 +24,7 @@ import ( "time" "github.com/aws/amazon-ecs-agent/agent/api/mocks" + "github.com/aws/amazon-ecs-agent/agent/config" "github.com/aws/amazon-ecs-agent/agent/eventstream" "github.com/aws/amazon-ecs-agent/agent/tcs/client" "github.com/aws/amazon-ecs-agent/agent/tcs/model/ecstcs" @@ -31,6 +32,7 @@ import ( "github.com/aws/amazon-ecs-agent/agent/wsclient/mock/utils" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/golang/mock/gomock" + "github.com/gorilla/websocket" "github.com/stretchr/testify/assert" "golang.org/x/net/context" ) @@ -46,6 +48,11 @@ const ( type mockStatsEngine struct{} +var testCfg = &config.Config{ + AcceptInsecureCert: true, + AWSRegion: "us-east-1", +} + func (engine *mockStatsEngine) GetInstanceMetrics() (*ecstcs.MetricsMetadata, []*ecstcs.TaskMetric, error) { req := createPublishMetricsRequest() return req.Metadata, req.TaskMetrics, nil @@ -74,7 +81,7 @@ func TestFormatURL(t *testing.T) { func TestStartSession(t *testing.T) { // Start test server. - closeWS := make(chan bool) + closeWS := make(chan []byte) server, serverChan, requestChan, serverErr, err := mockwsutils.StartMockServer(t, closeWS) defer server.Close() if err != nil { @@ -92,13 +99,13 @@ func TestStartSession(t *testing.T) { wait.Done() }() defer func() { - close(closeWS) + closeSocket(closeWS) close(serverChan) }() deregisterInstanceEventStream := eventstream.NewEventStream("Deregister_Instance", context.Background()) // Start a session with the test server. - go startSession(server.URL, "us-east-1", credentials.AnonymousCredentials, true, &mockStatsEngine{}, defaultHeartbeatTimeout, defaultHeartbeatJitter, testPublishMetricsInterval, deregisterInstanceEventStream) + go startSession(server.URL, testCfg, credentials.AnonymousCredentials, &mockStatsEngine{}, defaultHeartbeatTimeout, defaultHeartbeatJitter, testPublishMetricsInterval, deregisterInstanceEventStream) // startSession internally starts publishing metrics from the mockStatsEngine object. time.Sleep(testPublishMetricsInterval) @@ -120,7 +127,7 @@ func TestStartSession(t *testing.T) { } // Decode and verify the metric data. - _, responseType, err := wsclient.DecodeData([]byte(payload), &tcsclient.TcsDecoder{}) + _, responseType, err := wsclient.DecodeData([]byte(payload), tcsclient.NewTCSDecoder()) if err != nil { t.Fatal("error decoding data: ", err) } @@ -131,9 +138,9 @@ func TestStartSession(t *testing.T) { wait.Wait() } -func TestSessionConenctionClosedByRemote(t *testing.T) { +func TestSessionConnectionClosedByRemote(t *testing.T) { // Start test server. - closeWS := make(chan bool) + closeWS := make(chan []byte) server, serverChan, _, serverErr, err := mockwsutils.StartMockServer(t, closeWS) defer server.Close() if err != nil { @@ -141,14 +148,14 @@ func TestSessionConenctionClosedByRemote(t *testing.T) { } go func() { serr := <-serverErr - if serr != io.EOF { + if !websocket.IsCloseError(serr, websocket.CloseNormalClosure) { t.Error(serr) } }() sleepBeforeClose := 10 * time.Millisecond go func() { time.Sleep(sleepBeforeClose) - close(closeWS) + closeSocket(closeWS) close(serverChan) }() @@ -158,7 +165,7 @@ func TestSessionConenctionClosedByRemote(t *testing.T) { defer cancel() // Start a session with the test server. - err = startSession(server.URL, "us-east-1", credentials.AnonymousCredentials, true, &mockStatsEngine{}, defaultHeartbeatTimeout, defaultHeartbeatJitter, testPublishMetricsInterval, deregisterInstanceEventStream) + err = startSession(server.URL, testCfg, credentials.AnonymousCredentials, &mockStatsEngine{}, defaultHeartbeatTimeout, defaultHeartbeatJitter, testPublishMetricsInterval, deregisterInstanceEventStream) if err == nil { t.Error("Expected io.EOF on closed connection") @@ -172,7 +179,7 @@ func TestSessionConenctionClosedByRemote(t *testing.T) { // connection or it's inactive for too long func TestConnectionInactiveTimeout(t *testing.T) { // Start test server. - closeWS := make(chan bool) + closeWS := make(chan []byte) server, _, requestChan, serverErr, err := mockwsutils.StartMockServer(t, closeWS) defer server.Close() if err != nil { @@ -192,12 +199,13 @@ func TestConnectionInactiveTimeout(t *testing.T) { deregisterInstanceEventStream.StartListening() defer cancel() // Start a session with the test server. - err = startSession(server.URL, "us-east-1", credentials.AnonymousCredentials, true, &mockStatsEngine{}, 50*time.Millisecond, 100*time.Millisecond, testPublishMetricsInterval, deregisterInstanceEventStream) + err = startSession(server.URL, testCfg, credentials.AnonymousCredentials, &mockStatsEngine{}, 50*time.Millisecond, 100*time.Millisecond, testPublishMetricsInterval, deregisterInstanceEventStream) // if we are not blocked here, then the test pass as it will reconnect in StartSession assert.Error(t, err, "Close the connection should cause the tcs client return error") - assert.EqualError(t, <-serverErr, io.ErrUnexpectedEOF.Error(), "Read from closed connection should got io.UnexpectedEOF error") - close(closeWS) + assert.True(t, websocket.IsCloseError(<-serverErr, websocket.CloseAbnormalClosure), "Read from closed connection should produce an io.EOF error") + + closeSocket(closeWS) } func TestDiscoverEndpointAndStartSession(t *testing.T) { @@ -222,6 +230,13 @@ func getPayloadFromRequest(request string) (string, error) { return "", errors.New("Could not get payload") } +// closeSocket tells the server to send a close frame. This lets us test +// what happens if the connection is closed by the remote server. +func closeSocket(ws chan<- []byte) { + ws <- websocket.FormatCloseMessage(websocket.CloseNormalClosure, "") + close(ws) +} + func createPublishMetricsRequest() *ecstcs.PublishMetricsRequest { cluster := testClusterArn ci := testInstanceArn diff --git a/agent/wsclient/client.go b/agent/wsclient/client.go index f1061458f79..bc48adcd26b 100644 --- a/agent/wsclient/client.go +++ b/agent/wsclient/client.go @@ -1,4 +1,4 @@ -// Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the @@ -19,9 +19,7 @@ package wsclient import ( - "bufio" "crypto/tls" - "encoding/base64" "encoding/json" "fmt" "io" @@ -29,10 +27,12 @@ import ( "net" "net/http" "net/url" + "os" "reflect" - "strings" + "sync" "time" + "github.com/aws/amazon-ecs-agent/agent/config" "github.com/aws/amazon-ecs-agent/agent/utils" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/private/protocol/json/jsonutil" @@ -53,6 +53,12 @@ const ( // writeBufSize is the size of the write buffer for the ws connection. writeBufSize = 32768 + + // gorilla/websocket expects the websocket scheme (ws[s]://) + wsScheme = "wss" + + // Default NO_PROXY env var IP addresses + defaultNoProxyIP = "169.254.169.254,169.254.170.2" ) // ReceivedMessage is the intermediate message used to unmarshal a @@ -89,7 +95,10 @@ type ClientServer interface { // ClientServer SetAnyRequestHandler(RequestHandler) MakeRequest(input interface{}) error + WriteMessage(input []byte) error Connect() error + IsConnected() bool + SetConnection(conn WebsocketConn) Disconnect(...interface{}) error Serve() error io.Closer @@ -99,10 +108,12 @@ type ClientServer interface { // ClientServerImpl wraps commonly used methods defined in ClientServer interface. type ClientServerImpl struct { - AcceptInvalidCert bool - Conn WebsocketConn + // AgentConfig is the user-specified runtime configuration + AgentConfig *config.Config + // conn holds the underlying low-level websocket connection + conn WebsocketConn + // CredentialProvider is used to retrieve AWS credentials CredentialProvider *credentials.Credentials - Region string // RequestHandlers is a map from message types to handler functions of the // form: // "FooMessage": func(message *ecsacs.FooMessage) @@ -113,6 +124,8 @@ type ClientServerImpl struct { AnyRequestHandler RequestHandler // URL is the full url to the backend, including path, querystring, and so on. URL string + // writeLock needed to ensure that only one routine is writing to the socket + writeLock sync.Mutex ClientServer ServiceError TypeDecoder @@ -127,23 +140,45 @@ func (cs *ClientServerImpl) Connect() error { return err } + parsedURL.Scheme = wsScheme + // NewRequest never returns an error if the url parses and we just verified // it did above - request, _ := http.NewRequest("GET", cs.URL, nil) + request, _ := http.NewRequest("GET", parsedURL.String(), nil) + // Sign the request; we'll send its headers via the websocket client which includes the signature - utils.SignHTTPRequest(request, cs.Region, ServiceName, cs.CredentialProvider, nil) + utils.SignHTTPRequest(request, cs.AgentConfig.AWSRegion, ServiceName, cs.CredentialProvider, nil) - wsConn, err := cs.websocketConn(parsedURL, request) - if err != nil { - return err + timeoutDialer := &net.Dialer{Timeout: wsConnectTimeout} + tlsConfig := &tls.Config{ServerName: parsedURL.Host, InsecureSkipVerify: cs.AgentConfig.AcceptInsecureCert} + + // Ensure that NO_PROXY gets set + noProxy := os.Getenv("NO_PROXY") + if noProxy == "" { + dockerHost, err := url.Parse(cs.AgentConfig.DockerEndpoint) + if err == nil { + dockerHost.Scheme = "" + os.Setenv("NO_PROXY", fmt.Sprintf("%s,%s", defaultNoProxyIP, dockerHost.String())) + seelog.Info("NO_PROXY set:", os.Getenv("NO_PROXY")) + } else { + seelog.Errorf("NO_PROXY unable to be set: the configured Docker endpoint is invalid.") + } + } + + dialer := websocket.Dialer{ + ReadBufferSize: readBufSize, + WriteBufferSize: writeBufSize, + TLSClientConfig: tlsConfig, + Proxy: http.ProxyFromEnvironment, + NetDial: timeoutDialer.Dial, } - websocketConn, httpResponse, err := websocket.NewClient(wsConn, parsedURL, request.Header, readBufSize, writeBufSize) + websocketConn, httpResponse, err := dialer.Dial(parsedURL.String(), request.Header) if httpResponse != nil { defer httpResponse.Body.Close() } + if err != nil { - defer wsConn.Close() var resp []byte if httpResponse != nil { var readErr error @@ -161,14 +196,22 @@ func (cs *ClientServerImpl) Connect() error { seelog.Warnf("Error creating a websocket client: %v", err) return fmt.Errorf(string(resp) + ", " + err.Error()) } - cs.Conn = websocketConn + cs.conn = websocketConn return nil } +func (cs *ClientServerImpl) IsReady() bool { + return cs.conn != nil +} + +func (cs *ClientServerImpl) SetConnection(conn WebsocketConn) { + cs.conn = conn +} + // Disconnect disconnects the connection func (cs *ClientServerImpl) Disconnect(...interface{}) error { - if cs.Conn != nil { - return cs.Conn.Close() + if cs.conn != nil { + return cs.conn.Close() } return fmt.Errorf("No Connection to close") @@ -209,34 +252,43 @@ func (cs *ClientServerImpl) MakeRequest(input interface{}) error { // Over the wire we send something like // {"type":"AckRequest","message":{"messageId":"xyz"}} - return cs.Conn.WriteMessage(websocket.TextMessage, send) + return cs.WriteMessage(send) +} + +// WriteMessage wraps the low level websocket write method with a lock +func (cs *ClientServerImpl) WriteMessage(send []byte) error { + cs.writeLock.Lock() + defer cs.writeLock.Unlock() + return cs.conn.WriteMessage(websocket.TextMessage, send) } // ConsumeMessages reads messages from the websocket connection and handles read // messages from an active connection. func (cs *ClientServerImpl) ConsumeMessages() error { - var err error for { - messageType, message, cerr := cs.Conn.ReadMessage() - err = cerr - if err != nil { - if err != io.EOF && err != io.ErrUnexpectedEOF { - if message != nil { - seelog.Errorf("Error getting message from ws backend: %v, message: %v", err, string(message)) - } else { - seelog.Errorf("Error getting message from ws backend: %v", err) - } + messageType, message, err := cs.conn.ReadMessage() + + switch { + + case err == nil: + if messageType != websocket.TextMessage { + // maybe not fatal though, we'll try to process it anyways + seelog.Errorf("Unexpected messageType: %v", messageType) } - break - } - if messageType != websocket.TextMessage { - seelog.Errorf("Unexpected messageType: %s", messageType) - // maybe not fatal though, we'll try to process it anyways + seelog.Debug("Got a message from websocket") + cs.handleMessage(message) + + case permissibleCloseCode(err): + seelog.Warnf("Connection closed for a valid reason: %s", err) + return io.EOF + + default: + //Unexpected error occurred + seelog.Errorf("Error getting message from ws backend: error: [%v], message: [%s], messageType: [%v] ", err, message, messageType) + return err } - seelog.Debug("Got a message from websocket") - cs.handleMessage(message) + } - return err } // CreateRequestMessage creates the request json message using the given input. @@ -290,75 +342,7 @@ func (cs *ClientServerImpl) handleMessage(data []byte) { } } -// websocketConn establishes a connection to the given URL, respecting any proxy configuration in the environment. -// A standard proxying setup involves setting the following environment variables -// (may be listed in /etc/ecs/ecs.config if using ecs-init): -// HTTP_PROXY=http:/// # HTTPS_PROXY may be set instead or additionally -// NO_PROXY=169.254.169.254,/var/run/docker.sock # Directly connect to metadata service and docker socket -func (cs *ClientServerImpl) websocketConn(parsedURL *url.URL, request *http.Request) (net.Conn, error) { - proxyURL, err := http.ProxyFromEnvironment(request) - if err != nil { - return nil, err - } - - // url.Host might not have the port, but tls.Dial needs it - targetHost := parsedURL.Host - if !strings.Contains(targetHost, ":") { - targetHost += ":443" - } - targetHostname, _, err := net.SplitHostPort(targetHost) - if err != nil { - return nil, err - } - - tlsConfig := tls.Config{ServerName: targetHostname, InsecureSkipVerify: cs.AcceptInvalidCert} - timeoutDialer := &net.Dialer{Timeout: wsConnectTimeout} - - if proxyURL == nil { - // directly connect - seelog.Infof("Creating poll dialer, host: %s", parsedURL.Host) - return tls.DialWithDialer(timeoutDialer, "tcp", targetHost, &tlsConfig) - } - - // connect via proxy - seelog.Infof("Creating poll dialer, proxy: %s", proxyURL.Host) - plainConn, err := timeoutDialer.Dial("tcp", proxyURL.Host) - if err != nil { - return nil, err - } - - // TLS over an HTTP proxy via CONNECT taken from: https://golang.org/src/net/http/transport.go - connectReq := &http.Request{ - Method: "CONNECT", - URL: &url.URL{Opaque: targetHost}, - Host: targetHost, - Header: make(http.Header), - } - - if proxyUser := proxyURL.User; proxyUser != nil { - username := proxyUser.Username() - password, _ := proxyUser.Password() - auth := base64.StdEncoding.EncodeToString([]byte(username + ":" + password)) - connectReq.Header.Set("Proxy-Authorization", "Basic "+auth) - } - - connectReq.Write(plainConn) - - // Read response. - // Okay to use and discard buffered reader here, because - // TLS server will not speak until spoken to. - br := bufio.NewReader(plainConn) - resp, err := http.ReadResponse(br, connectReq) - if err != nil { - plainConn.Close() - return nil, err - } - if resp.StatusCode != 200 { - plainConn.Close() - return nil, fmt.Errorf(resp.Status) - } - - tlsConn := tls.Client(plainConn, &tlsConfig) - - return tlsConn, nil +// See https://github.com/gorilla/websocket/blob/87f6f6a22ebfbc3f89b9ccdc7fddd1b914c095f9/conn.go#L650 +func permissibleCloseCode(err error) bool { + return websocket.IsCloseError(err, websocket.CloseNormalClosure, websocket.CloseGoingAway, websocket.CloseInternalServerErr) } diff --git a/agent/wsclient/client_test.go b/agent/wsclient/client_test.go new file mode 100644 index 00000000000..f9cc4241208 --- /dev/null +++ b/agent/wsclient/client_test.go @@ -0,0 +1,151 @@ +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package wsclient + +import ( + "io" + "os" + "testing" + + "github.com/aws/amazon-ecs-agent/agent/acs/model/ecsacs" + "github.com/aws/amazon-ecs-agent/agent/config" + "github.com/aws/amazon-ecs-agent/agent/wsclient/mock/utils" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + + "github.com/gorilla/websocket" + + "github.com/stretchr/testify/assert" +) + +const dockerEndpoint = "/var/run/docker.sock" + +// TestConcurrentWritesDontPanic will force a panic in the websocket library if +// the implemented methods don't handle concurrency correctly +// See https://godoc.org/github.com/gorilla/websocket#hdr-Concurrency +func TestConcurrentWritesDontPanic(t *testing.T) { + closeWS := make(chan []byte) + defer close(closeWS) + + mockServer, _, requests, _, _ := mockwsutils.StartMockServer(t, closeWS) + defer mockServer.Close() + + req := ecsacs.AckRequest{Cluster: aws.String("test"), ContainerInstance: aws.String("test"), MessageId: aws.String("test")} + + cs := getClientServer(mockServer.URL) + cs.Connect() + + executeTenRequests := func() { + for i := 0; i < 10; i++ { + cs.MakeRequest(&req) + } + } + + // Make requests from two separate routines to try and force a + // concurrent write + go executeTenRequests() + executeTenRequests() + + t.Log("Waiting for all 20 requests to succeed") + for i := 0; i < 20; i++ { + <-requests + } +} + +// TestProxyVariableCustomValue ensures that a user is able to override the +// proxy variable by setting an environment variable +func TestProxyVariableCustomValue(t *testing.T) { + closeWS := make(chan []byte) + defer close(closeWS) + + mockServer, _, _, _, _ := mockwsutils.StartMockServer(t, closeWS) + defer mockServer.Close() + + testString := "Custom no proxy string" + os.Setenv("NO_PROXY", testString) + getClientServer(mockServer.URL).Connect() + + assert.Equal(t, os.Getenv("NO_PROXY"), testString, "NO_PROXY should match user-supplied variable") +} + +// TestProxyVariableDefaultValue verifies that NO_PROXY gets overridden if it +// isn't already set +func TestProxyVariableDefaultValue(t *testing.T) { + closeWS := make(chan []byte) + defer close(closeWS) + + mockServer, _, _, _, _ := mockwsutils.StartMockServer(t, closeWS) + defer mockServer.Close() + + os.Unsetenv("NO_PROXY") + getClientServer(mockServer.URL).Connect() + + expectedEnvVar := "169.254.169.254,169.254.170.2," + dockerEndpoint + + assert.Equal(t, os.Getenv("NO_PROXY"), expectedEnvVar, "Variable NO_PROXY expected to be overwritten when no default value supplied") +} + +// TestHandleMessagePermissibleCloseCode ensures that permissible close codes +// are wrapped in io.EOF +func TestHandleMessagePermissibleCloseCode(t *testing.T) { + closeWS := make(chan []byte) + defer close(closeWS) + + messageError := make(chan error) + mockServer, _, _, _, _ := mockwsutils.StartMockServer(t, closeWS) + cs := getClientServer(mockServer.URL) + cs.Connect() + + go func() { + messageError <- cs.ConsumeMessages() + }() + + closeWS <- websocket.FormatCloseMessage(websocket.CloseNormalClosure, ":)") + assert.EqualError(t, <-messageError, io.EOF.Error(), "expected EOF for normal close code") +} + +// TestHandleMessageUnexpectedCloseCode checks that unexpected close codes will +// be returned as is (not wrapped in io.EOF) +func TestHandleMessageUnexpectedCloseCode(t *testing.T) { + closeWS := make(chan []byte) + defer close(closeWS) + + messageError := make(chan error) + mockServer, _, _, _, _ := mockwsutils.StartMockServer(t, closeWS) + cs := getClientServer(mockServer.URL) + cs.Connect() + + go func() { + messageError <- cs.ConsumeMessages() + }() + + closeWS <- websocket.FormatCloseMessage(websocket.CloseTryAgainLater, ":(") + assert.True(t, websocket.IsCloseError(<-messageError, websocket.CloseTryAgainLater), "Expected error from websocket library") +} + +func getClientServer(url string) *ClientServerImpl { + types := []interface{}{ecsacs.AckRequest{}} + + return &ClientServerImpl{ + URL: url, + AgentConfig: &config.Config{ + AcceptInsecureCert: true, + AWSRegion: "us-east-1", + DockerEndpoint: "unix://" + dockerEndpoint, + }, + CredentialProvider: credentials.AnonymousCredentials, + TypeDecoder: BuildTypeDecoder(types), + } +} diff --git a/agent/wsclient/mock/client.go b/agent/wsclient/mock/client.go index 2eb7419f75d..1f5246d49fe 100644 --- a/agent/wsclient/mock/client.go +++ b/agent/wsclient/mock/client.go @@ -1,4 +1,4 @@ -// Copyright 2015-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2015-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the @@ -84,6 +84,16 @@ func (_mr *_MockClientServerRecorder) Disconnect(arg0 ...interface{}) *gomock.Ca return _mr.mock.ctrl.RecordCall(_mr.mock, "Disconnect", arg0...) } +func (_m *MockClientServer) IsConnected() bool { + ret := _m.ctrl.Call(_m, "IsConnected") + ret0, _ := ret[0].(bool) + return ret0 +} + +func (_mr *_MockClientServerRecorder) IsConnected() *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "IsConnected") +} + func (_m *MockClientServer) MakeRequest(_param0 interface{}) error { ret := _m.ctrl.Call(_m, "MakeRequest", _param0) ret0, _ := ret[0].(error) @@ -111,3 +121,21 @@ func (_m *MockClientServer) SetAnyRequestHandler(_param0 wsclient.RequestHandler func (_mr *_MockClientServerRecorder) SetAnyRequestHandler(arg0 interface{}) *gomock.Call { return _mr.mock.ctrl.RecordCall(_mr.mock, "SetAnyRequestHandler", arg0) } + +func (_m *MockClientServer) SetConnection(_param0 wsclient.WebsocketConn) { + _m.ctrl.Call(_m, "SetConnection", _param0) +} + +func (_mr *_MockClientServerRecorder) SetConnection(arg0 interface{}) *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "SetConnection", arg0) +} + +func (_m *MockClientServer) WriteMessage(_param0 []byte) error { + ret := _m.ctrl.Call(_m, "WriteMessage", _param0) + ret0, _ := ret[0].(error) + return ret0 +} + +func (_mr *_MockClientServerRecorder) WriteMessage(arg0 interface{}) *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "WriteMessage", arg0) +} diff --git a/agent/wsclient/mock/utils/utils.go b/agent/wsclient/mock/utils/utils.go index 449e940250f..0fa1dc4c839 100644 --- a/agent/wsclient/mock/utils/utils.go +++ b/agent/wsclient/mock/utils/utils.go @@ -1,4 +1,4 @@ -// Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2015-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the @@ -23,17 +23,18 @@ import ( ) // StartMockServer starts a mock websocket server. -func StartMockServer(t *testing.T, closeWS <-chan bool) (*httptest.Server, chan<- string, <-chan string, <-chan error, error) { +func StartMockServer(t *testing.T, closeWS <-chan []byte) (*httptest.Server, chan<- string, <-chan string, <-chan error, error) { serverChan := make(chan string) requestsChan := make(chan string) errChan := make(chan error) + stopListen := make(chan bool) upgrader := websocket.Upgrader{ReadBufferSize: 1024, WriteBufferSize: 1024} handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ws, err := upgrader.Upgrade(w, r, nil) go func() { - <-closeWS - ws.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""), time.Now().Add(time.Second)) + ws.WriteControl(websocket.CloseMessage, <-closeWS, time.Now().Add(time.Second)) + close(stopListen) }() if err != nil { errChan <- err @@ -41,7 +42,7 @@ func StartMockServer(t *testing.T, closeWS <-chan bool) (*httptest.Server, chan< go func() { for { select { - case <-closeWS: + case <-stopListen: return default: _, msg, err := ws.ReadMessage() diff --git a/agent/wsclient/types.go b/agent/wsclient/types.go index a2dd69d4c4b..eb8e651f78f 100644 --- a/agent/wsclient/types.go +++ b/agent/wsclient/types.go @@ -1,4 +1,4 @@ -// Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the @@ -27,3 +27,33 @@ type TypeDecoder interface { // be marshalled/unmarshalled to/from. GetRecognizedTypes() map[string]reflect.Type } + +// TypeDecoderImpl is an implementation for general use between ACS and +// TCS clients +type TypeDecoderImpl struct { + typeMappings map[string]reflect.Type +} + +// BuildTypeDecoder takes a list of interfaces and stores them internally as a +// list of typeMappings in the format below. +// "MyMessage": TypeOf(ecstcs.MyMessage) +func BuildTypeDecoder(recognizedTypes []interface{}) TypeDecoder { + typeMappings := make(map[string]reflect.Type) + for _, recognizedType := range recognizedTypes { + typeMappings[reflect.TypeOf(recognizedType).Name()] = reflect.TypeOf(recognizedType) + } + + return &TypeDecoderImpl{typeMappings: typeMappings} +} + +func (d *TypeDecoderImpl) NewOfType(typeString string) (interface{}, bool) { + rtype, ok := d.typeMappings[typeString] + if !ok { + return nil, false + } + return reflect.New(rtype).Interface(), true +} + +func (d *TypeDecoderImpl) GetRecognizedTypes() map[string]reflect.Type { + return d.typeMappings +} From 496df868a389e5f09464a499e912658de2070d47 Mon Sep 17 00:00:00 2001 From: Samuel Karp Date: Wed, 8 Feb 2017 16:15:59 -0800 Subject: [PATCH 11/27] api/task.go: Fix many metalinter warnings --- agent/acs/handler/acs_handler_test.go | 2 +- agent/api/container_test.go | 4 +- agent/api/ecsclient/client.go | 2 +- agent/api/ecsclient/client_test.go | 6 +- agent/api/port_binding.go | 2 +- agent/api/port_binding_test.go | 6 +- agent/api/task.go | 14 +- agent/api/task_test.go | 16 +- agent/api/testutils/container_equal.go | 2 +- agent/api/testutils/container_equal_test.go | 4 +- agent/api/types.go | 170 ++++++++++++++---- agent/engine/docker_container_engine_test.go | 4 +- .../engine/docker_image_manager_integ_test.go | 12 +- agent/engine/docker_task_engine.go | 18 +- agent/engine/dockerauth/ecr.go | 2 +- agent/engine/dockerauth/ecr_test.go | 22 +-- .../dockerstate/docker_task_engine_state.go | 12 +- agent/engine/dockerstate/dockerstate_test.go | 10 +- .../engine/dockerstate/testutils/json_test.go | 2 +- agent/engine/engine_unix_integ_test.go | 6 +- agent/engine/engine_windows_integ_test.go | 2 +- agent/handlers/v1_handlers.go | 2 +- agent/handlers/v1_handlers_test.go | 2 +- agent/stats/container_test.go | 4 +- agent/stats/engine_integ_test.go | 6 +- 25 files changed, 215 insertions(+), 117 deletions(-) diff --git a/agent/acs/handler/acs_handler_test.go b/agent/acs/handler/acs_handler_test.go index 12726677f2b..2ca46863bea 100644 --- a/agent/acs/handler/acs_handler_test.go +++ b/agent/acs/handler/acs_handler_test.go @@ -1103,7 +1103,7 @@ func validateAddedContainer(expectedContainer *api.Container, addedContainer *ap // fields that we are intrested in for comparison containerToCompareFromAdded := &api.Container{ Name: addedContainer.Name, - Cpu: addedContainer.Cpu, + CPU: addedContainer.CPU, Essential: addedContainer.Essential, Memory: addedContainer.Memory, Image: addedContainer.Image, diff --git a/agent/api/container_test.go b/agent/api/container_test.go index 7123e0fbdc9..68387fe6c84 100644 --- a/agent/api/container_test.go +++ b/agent/api/container_test.go @@ -26,7 +26,7 @@ func TestOverridden(t *testing.T) { Name: "name", Image: "image", Command: []string{"foo", "bar"}, - Cpu: 1, + CPU: 1, Memory: 1, Links: []string{}, Ports: []PortBinding{PortBinding{10, 10, "", TransportProtocolTCP}}, @@ -63,7 +63,7 @@ func (pair configPair) Equal() bool { if (conf.Memory / 1024 / 1024) != int64(cont.Memory) { return false } - if conf.CPUShares != int64(cont.Cpu) { + if conf.CPUShares != int64(cont.CPU) { return false } if conf.Image != cont.Image { diff --git a/agent/api/ecsclient/client.go b/agent/api/ecsclient/client.go index 7cb3b78a917..c61570ee36a 100644 --- a/agent/api/ecsclient/client.go +++ b/agent/api/ecsclient/client.go @@ -336,7 +336,7 @@ func (client *APIECSClient) SubmitContainerStateChange(change api.ContainerState for i, binding := range change.PortBindings { hostPort := int64(binding.HostPort) containerPort := int64(binding.ContainerPort) - bindIP := binding.BindIp + bindIP := binding.BindIP protocol := binding.Protocol.String() networkBindings[i] = &ecs.NetworkBinding{ BindIP: &bindIP, diff --git a/agent/api/ecsclient/client_test.go b/agent/api/ecsclient/client_test.go index 725cff112a1..651465fa7c2 100644 --- a/agent/api/ecsclient/client_test.go +++ b/agent/api/ecsclient/client_test.go @@ -116,12 +116,12 @@ func TestSubmitContainerStateChange(t *testing.T) { Status: api.ContainerRunning, PortBindings: []api.PortBinding{ api.PortBinding{ - BindIp: "1.2.3.4", + BindIP: "1.2.3.4", ContainerPort: 1, HostPort: 2, }, api.PortBinding{ - BindIp: "2.2.3.4", + BindIP: "2.2.3.4", ContainerPort: 3, HostPort: 4, Protocol: api.TransportProtocolUDP, @@ -247,7 +247,7 @@ func buildAttributeList(capabilities []string, attributes map[string]string) []* func TestReRegisterContainerInstance(t *testing.T) { additionalAttributes := map[string]string{"my_custom_attribute": "Custom_Value1", - "my_other_custom_attribute": "Custom_Value2", + "my_other_custom_attribute": "Custom_Value2", "attribute_name_with_no_value": "", } diff --git a/agent/api/port_binding.go b/agent/api/port_binding.go index 531f93dc54c..29da63e115e 100644 --- a/agent/api/port_binding.go +++ b/agent/api/port_binding.go @@ -47,7 +47,7 @@ func PortBindingFromDockerPortBinding(dockerPortBindings map[docker.Port][]docke portBindings = append(portBindings, PortBinding{ ContainerPort: uint16(containerPort), HostPort: uint16(hostPort), - BindIp: binding.HostIP, + BindIP: binding.HostIP, Protocol: protocol, }) } diff --git a/agent/api/port_binding_test.go b/agent/api/port_binding_test.go index 9b9657e2fa6..6da97b252b8 100644 --- a/agent/api/port_binding_test.go +++ b/agent/api/port_binding_test.go @@ -31,7 +31,7 @@ func TestPortBindingFromDockerPortBinding(t *testing.T) { }, []PortBinding{ PortBinding{ - BindIp: "1.2.3.4", + BindIP: "1.2.3.4", HostPort: 55, ContainerPort: 53, Protocol: TransportProtocolUDP, @@ -47,13 +47,13 @@ func TestPortBindingFromDockerPortBinding(t *testing.T) { }, []PortBinding{ PortBinding{ - BindIp: "2.3.4.5", + BindIP: "2.3.4.5", HostPort: 8080, ContainerPort: 80, Protocol: TransportProtocolTCP, }, PortBinding{ - BindIp: "5.6.7.8", + BindIP: "5.6.7.8", HostPort: 80, ContainerPort: 80, Protocol: TransportProtocolTCP, diff --git a/agent/api/task.go b/agent/api/task.go index 5504ad1f935..4b727bcb651 100644 --- a/agent/api/task.go +++ b/agent/api/task.go @@ -265,7 +265,7 @@ func (task *Task) dockerConfig(container *Container) (*docker.Config, *DockerCli Volumes: dockerVolumes, Env: dockerEnv, Memory: dockerMem, - CPUShares: task.dockerCpuShares(container.Cpu), + CPUShares: task.dockerCpuShares(container.CPU), } if container.DockerConfig.Config != nil { @@ -545,17 +545,17 @@ func (task *Task) updateKnownStatusTime() { } func (task *Task) SetCredentialsId(id string) { - task.credentialsIdLock.Lock() - defer task.credentialsIdLock.Unlock() + task.credentialsIDLock.Lock() + defer task.credentialsIDLock.Unlock() - task.credentialsId = id + task.credentialsID = id } func (task *Task) GetCredentialsId() string { - task.credentialsIdLock.RLock() - defer task.credentialsIdLock.RUnlock() + task.credentialsIDLock.RLock() + defer task.credentialsIDLock.RUnlock() - return task.credentialsId + return task.credentialsID } func (task *Task) GetDesiredStatus() TaskStatus { diff --git a/agent/api/task_test.go b/agent/api/task_test.go index c9ea54262f2..32c3007a26f 100644 --- a/agent/api/task_test.go +++ b/agent/api/task_test.go @@ -33,7 +33,7 @@ func strptr(s string) *string { return &s } func dockerMap(task *Task) map[string]*DockerContainer { m := make(map[string]*DockerContainer) for _, c := range task.Containers { - m[c.Name] = &DockerContainer{DockerId: "dockerid-" + c.Name, DockerName: "dockername-" + c.Name, Container: c} + m[c.Name] = &DockerContainer{DockerID: "dockerid-" + c.Name, DockerName: "dockername-" + c.Name, Container: c} } return m } @@ -84,7 +84,7 @@ func TestDockerConfigCPUShareZero(t *testing.T) { Containers: []*Container{ &Container{ Name: "c1", - Cpu: 0, + CPU: 0, }, }, } @@ -104,7 +104,7 @@ func TestDockerConfigCPUShareMinimum(t *testing.T) { Containers: []*Container{ &Container{ Name: "c1", - Cpu: 1, + CPU: 1, }, }, } @@ -124,7 +124,7 @@ func TestDockerConfigCPUShareUnchanged(t *testing.T) { Containers: []*Container{ &Container{ Name: "c1", - Cpu: 100, + CPU: 100, }, }, } @@ -257,7 +257,7 @@ func TestDockerHostConfigRawConfigMerging(t *testing.T) { &Container{ Name: "c1", Image: "image", - Cpu: 50, + CPU: 50, Memory: 100, VolumesFrom: []VolumeFrom{VolumeFrom{SourceContainer: "c2"}}, DockerConfig: DockerConfig{ @@ -394,7 +394,7 @@ func TestDockerConfigRawConfigMerging(t *testing.T) { &Container{ Name: "c1", Image: "image", - Cpu: 50, + CPU: 50, Memory: 100, DockerConfig: DockerConfig{ Config: strptr(string(rawConfig)), @@ -456,7 +456,7 @@ func TestGetCredentialsEndpointWhenCredentialsAreSet(t *testing.T) { Name: "c2", Environment: make(map[string]string), }}, - credentialsId: credentialsIDInTask, + credentialsID: credentialsIDInTask, } taskCredentials := &credentials.TaskIAMRoleCredentials{ @@ -652,7 +652,7 @@ func TestTaskFromACS(t *testing.T) { EntryPoint: &[]string{"sh", "-c"}, Essential: true, Environment: map[string]string{"key": "value"}, - Cpu: 10, + CPU: 10, Memory: 100, MountPoints: []MountPoint{ MountPoint{ diff --git a/agent/api/testutils/container_equal.go b/agent/api/testutils/container_equal.go index 9bc5147e17a..aeea7e3344e 100644 --- a/agent/api/testutils/container_equal.go +++ b/agent/api/testutils/container_equal.go @@ -38,7 +38,7 @@ func ContainersEqual(lhs, rhs *api.Container) bool { if !utils.StrSliceEqual(lhs.Command, rhs.Command) { return false } - if lhs.Cpu != rhs.Cpu || lhs.Memory != rhs.Memory { + if lhs.CPU != rhs.CPU || lhs.Memory != rhs.Memory { return false } // Order doesn't matter diff --git a/agent/api/testutils/container_equal_test.go b/agent/api/testutils/container_equal_test.go index c533dd4f2de..dd2cfa690c3 100644 --- a/agent/api/testutils/container_equal_test.go +++ b/agent/api/testutils/container_equal_test.go @@ -30,7 +30,7 @@ func TestContainerEqual(t *testing.T) { Container{Name: "name"}, Container{Name: "name"}, Container{Image: "nginx"}, Container{Image: "nginx"}, Container{Command: []string{"c"}}, Container{Command: []string{"c"}}, - Container{Cpu: 1}, Container{Cpu: 1}, + Container{CPU: 1}, Container{CPU: 1}, Container{Memory: 1}, Container{Memory: 1}, Container{Links: []string{"1", "2"}}, Container{Links: []string{"1", "2"}}, Container{Links: []string{"1", "2"}}, Container{Links: []string{"2", "1"}}, @@ -53,7 +53,7 @@ func TestContainerEqual(t *testing.T) { Container{Image: "nginx"}, Container{Image: "えんじんえっくす"}, Container{Command: []string{"c"}}, Container{Command: []string{"し"}}, Container{Command: []string{"c", "b"}}, Container{Command: []string{"b", "c"}}, - Container{Cpu: 1}, Container{Cpu: 2e2}, + Container{CPU: 1}, Container{CPU: 2e2}, Container{Memory: 1}, Container{Memory: 2e2}, Container{Links: []string{"1", "2"}}, Container{Links: []string{"1", "二"}}, Container{VolumesFrom: []VolumeFrom{VolumeFrom{"1", false}, VolumeFrom{"2", true}}}, Container{VolumesFrom: []VolumeFrom{VolumeFrom{"1", false}, VolumeFrom{"二", false}}}, diff --git a/agent/api/types.go b/agent/api/types.go index b1e468b1ed9..a4a22f4d545 100644 --- a/agent/api/types.go +++ b/agent/api/types.go @@ -21,40 +21,62 @@ import ( "time" ) +// TaskStatus is an enumeration of valid states in the task lifecycle type TaskStatus int32 const ( + // TaskStatusNone is the zero state of a task; this task has been received but no further progress has completed TaskStatusNone TaskStatus = iota + // TaskPulled represents a task which has had all its container images pulled, but not all have yet progressed passed pull TaskPulled + // TaskCreated represents a task which has had all its containers created TaskCreated + // TaskRunning represents a task which has had all its containers started TaskRunning + // TaskStopped represents a task in which all containers are stopped TaskStopped ) +// ContainerStatus is an enumeration of valid states in the container lifecycle type ContainerStatus int32 const ( + // ContainerStatusNone is the zero state of a container; this container has not completed pull ContainerStatusNone ContainerStatus = iota + // ContainerPulled represents a container which has had the image pulled ContainerPulled + // ContainerCreated represents a container that has been created ContainerCreated + // ContainerRunning represents a container that has started ContainerRunning + // ContainerStopped represents a container that has stopped ContainerStopped - ContainerZombie // Impossible status to use as a virtual 'max' + // ContainerZombie is an "impossible" state that is used as the maximum + ContainerZombie ) +// TransportProtocol is an enumeration of valid transport protocols type TransportProtocol int32 const ( + // TransportProtocolTCP represents TCP TransportProtocolTCP TransportProtocol = iota + // TransportProtocolUDP represents UDP TransportProtocolUDP ) +const ( + tcp = "tcp" + udp = "udp" +) + +// NewTransportProtocol returns a TransportProtocol from a string in the task func NewTransportProtocol(protocol string) (TransportProtocol, error) { switch protocol { - case "tcp": + case tcp: return TransportProtocolTCP, nil - case "udp": + case udp: return TransportProtocolUDP, nil default: return TransportProtocolTCP, errors.New(protocol + " is not a recognized transport protocol") @@ -63,54 +85,85 @@ func NewTransportProtocol(protocol string) (TransportProtocol, error) { func (tp *TransportProtocol) String() string { if tp == nil { - return "tcp" + return tcp } switch *tp { case TransportProtocolUDP: - return "udp" + return udp case TransportProtocolTCP: - return "tcp" + return tcp default: log.Crit("Unknown TransportProtocol type!") - return "tcp" + return tcp } } +// PortBinding represents a port binding for a container type PortBinding struct { + // ContainerPort is the port inside the container ContainerPort uint16 - HostPort uint16 - BindIp string - Protocol TransportProtocol + // HostPort is the port exposed on the host + HostPort uint16 + // BindIP is the IP address to which the port is bound + BindIP string `json:"BindIp"` + // Protocol is the protocol of the port + Protocol TransportProtocol } +// TaskOverrides are the overrides applied to a task type TaskOverrides struct{} +// Task is the internal representation of a task in the ECS agent type Task struct { - Arn string - Overrides TaskOverrides `json:"-"` - Family string - Version string + // Arn is the unique identifer for the task + Arn string + // Overrides are the overrides applied to a task + Overrides TaskOverrides `json:"-"` + // Family is the name of the task definition family + Family string + // Version is the version of the task definition + Version string + // Containers are the containers for the task Containers []*Container - Volumes []TaskVolume `json:"volumes"` - + // Volumes are the volumes for the task + Volumes []TaskVolume `json:"volumes"` + + // DesiredStatus represents the state where the task should go. Generally the desired status is informed by the ECS + // backend as a result of either API calls made to ECS or decisions made by the ECS service scheduler. The + // DesiredStatus is almost always either TaskRunning or TaskStopped. Do not access DesiredStatus directly. Instead, + // use `UpdateStatus`, `UpdateDesiredStatus`, `SetDesiredStatus`, and `SetDesiredStatus`. + // TODO DesiredStatus should probably be private with appropriately written setter/getter. When this is done, we need + // to ensure that the UnmarshalJSON is handled properly so that the state storage continues to work. DesiredStatus TaskStatus desiredStatusLock sync.RWMutex - KnownStatus TaskStatus - knownStatusLock sync.RWMutex + // KnownStatus represents the state where the task is. This is generally the minimum of equivalent status types for + // the containers in the task; if one container is at ContainerRunning and another is at ContainerPulled, the task + // KnownStatus would be TaskPulled. Do not access KnownStatus directly. Instead, use `UpdateStatus`, + // `UpdateKnownStatusAndTime`, and `GetKnownStatus`. + // TODO KnownStatus should probably be private with appropriately written setter/getter. When this is done, we need + // to ensure that the UnmarshalJSON is handled properly so that the state storage continues to work. + KnownStatus TaskStatus + knownStatusLock sync.RWMutex + // KnownStatusTime captures the time when the KnownStatus was last updated. Do not access KnownStatusTime directly, + // instead use `GetKnownStatusTime`. KnownStatusTime time.Time `json:"KnownTime"` knownStatusTimeLock sync.RWMutex + // SentStatus represents the last KnownStatus that was sent to the ECS SubmitTaskStateChange API. + // TODO(samuelkarp) SentStatus needs a lock and setters/getters. + // TODO SentStatus should probably be private with appropriately written setter/getter. When this is done, we need + // to ensure that the UnmarshalJSON is handled properly so that the state storage continues to work. SentStatus TaskStatus StartSequenceNumber int64 StopSequenceNumber int64 - // credentialsId is used to set the CredentialsId field for the + // credentialsID is used to set the CredentialsId field for the // IAMRoleCredentials object associated with the task. This id can be // used to look up the credentials for task in the credentials manager - credentialsId string - credentialsIdLock sync.RWMutex + credentialsID string + credentialsIDLock sync.RWMutex } // TaskVolume is a definition of all the volumes available for containers to @@ -145,26 +198,38 @@ func (fs *FSHostVolume) SourcePath() string { return fs.FSSourcePath } +// EmptyHostVolume represents a volume without a specified host path type EmptyHostVolume struct { HostPath string `json:"hostPath"` } +// SourcePath returns the generated host path for the volume func (e *EmptyHostVolume) SourcePath() string { return e.HostPath } +// ContainerStateChange represents a state change that needs to be sent to the +// SubmitContainerStateChange API type ContainerStateChange struct { - TaskArn string + // TaskArn is the unique identifier for the task + TaskArn string + // ContainerName is the name of the container ContainerName string - Status ContainerStatus - - Reason string - ExitCode *int + // Status is the status to send + Status ContainerStatus + + // Reason may contain details of why the container stopped + Reason string + // ExitCode is the exit code of the container, if available + ExitCode *int + // PortBindings are the details of the host ports picked for the specified + // container ports PortBindings []PortBinding // This bit is a little hacky; a pointer to the container's sentstatus which // may be updated to indicate what status was sent. This is used to ensure // the same event is handled only once. + // TODO(samuelkarp) Change this to expose a *Container and use container.UpdateSentStatus SentStatus *ContainerStatus } @@ -185,15 +250,21 @@ func (c *ContainerStateChange) String() string { return res } +// TaskStateChange represents a state change that needs to be sent to the +// SubmitTaskStateChange API type TaskStateChange struct { + // TaskArn is the unique identifier for the task TaskArn string - Status TaskStatus - Reason string + // Status is the status to send + Status TaskStatus + // Reason may contain details of why the task stopped + Reason string // As above, this is the same sort of hacky. // This is a pointer to the task's sent-status that gives the event handler a // hook into storing metadata about the task on the task such that it follows // the lifecycle of the task and so on. + // TODO(samuelkarp) Change this to expose a *Task and use task.UpdateSentStatus SentStatus *TaskStatus } @@ -214,16 +285,22 @@ func (t *Task) String() string { return res + "]" } +// ContainerOverrides are overrides applied to the container type ContainerOverrides struct { Command *[]string `json:"command"` } +// Container is the internal representation of a container in the ECS agent type Container struct { - Name string - Image string - ImageID string + // Name is the name of the container specified in the task definition + Name string + // Image is the image name specified in the task definition + Image string + // ImageID is the local ID of the image used in the container + ImageID string + Command []string - Cpu uint + CPU uint `json:"Cpu"` Memory uint Links []string VolumesFrom []VolumeFrom `json:"volumesFrom"` @@ -236,9 +313,20 @@ type Container struct { DockerConfig DockerConfig `json:"dockerConfig"` RegistryAuthentication *RegistryAuthenticationData `json:"registryAuthentication"` + // DesiredStatus represents the state where the container should go. Generally the desired status is informed by the + // ECS backend as a result of either API calls made to ECS or decisions made by the ECS service scheduler, though the + // agent may also set the DesiredStatus if a different "essential" container in the task exits. The DesiredStatus is + // almost always either ContainerRunning or ContainerStopped. Do not access DesiredStatus directly. Instead, + // use `GetDesiredStatus` and `SetDesiredStatus`. + // TODO DesiredStatus should probably be private with appropriately written setter/getter. When this is done, we need + // to ensure that the UnmarshalJSON is handled properly so that the state storage continues to work. DesiredStatus ContainerStatus `json:"desiredStatus"` desiredStatusLock sync.RWMutex + // KnownStatus represents the state where the container is. Do not access KnownStatus directly. Instead, use + // `GetKnownStatus` and `SetKnownStatus`. + // TODO KnownStatus should probably be private with appropriately written setter/getter. When this is done, we need + // to ensure that the UnmarshalJSON is handled properly so that the state storage continues to work. KnownStatus ContainerStatus knownStatusLock sync.RWMutex @@ -248,11 +336,17 @@ type Container struct { // 'Internal' containers are ones that are not directly specified by task definitions, but created by the agent IsInternal bool + // AppliedStatus is the status that has been "applied" (e.g., we've called Pull, Create, Start, or Stop) but we don't + // yet know that the application was successful. AppliedStatus ContainerStatus // ApplyingError is an error that occured trying to transition the container to its desired state // It is propagated to the backend in the form 'Name: ErrorString' as the 'reason' field. ApplyingError *DefaultNamedError + // SentStatus represents the last KnownStatus that was sent to the ECS SubmitContainerStateChange API. + // TODO(samuelkarp) SentStatus needs a lock and setters/getters. + // TODO SentStatus should probably be private with appropriately written setter/getter. When this is done, we need + // to ensure that the UnmarshalJSON is handled properly so that the state storage continues to work. SentStatus ContainerStatus KnownExitCode *int @@ -277,15 +371,18 @@ type VolumeFrom struct { ReadOnly bool `json:"readOnly"` } +// RegistryAuthenticationData is the authentication data sent by the ECS backend. Currently, the only supported +// authentication data is for ECR. type RegistryAuthenticationData struct { Type string `json:"type"` ECRAuthData *ECRAuthData `json:"ecrAuthData"` } +// ECRAuthData is the authentication details for ECR specifying the region, registryID, and possible endpoint override type ECRAuthData struct { EndpointOverride string `json:"endpointOverride"` Region string `json:"region"` - RegistryId string `json:"registryId"` + RegistryID string `json:"registryId"` } func (c *Container) String() string { @@ -296,6 +393,7 @@ func (c *Container) String() string { return ret } +// Resource is an on-host resource type Resource struct { Name string Type string @@ -303,12 +401,12 @@ type Resource struct { LongValue int64 } -// This is a mapping between containers-as-docker-knows-them and +// DockerContainer is a mapping between containers-as-docker-knows-them and // containers-as-we-know-them. // This is primarily used in DockerState, but lives here such that tasks and // containers know how to convert themselves into Docker's desired config format type DockerContainer struct { - DockerId string + DockerID string `json:"DockerId"` DockerName string // needed for linking Container *Container @@ -318,5 +416,5 @@ func (dc *DockerContainer) String() string { if dc == nil { return "nil" } - return fmt.Sprintf("Id: %s, Name: %s, Container: %s", dc.DockerId, dc.DockerName, dc.Container.String()) + return fmt.Sprintf("Id: %s, Name: %s, Container: %s", dc.DockerID, dc.DockerName, dc.Container.String()) } diff --git a/agent/engine/docker_container_engine_test.go b/agent/engine/docker_container_engine_test.go index 5d9b87c3db5..b1e827dc939 100644 --- a/agent/engine/docker_container_engine_test.go +++ b/agent/engine/docker_container_engine_test.go @@ -257,7 +257,7 @@ func TestPullImageECRSuccess(t *testing.T) { authData := &api.RegistryAuthenticationData{ Type: "ecr", ECRAuthData: &api.ECRAuthData{ - RegistryId: registryID, + RegistryID: registryID, Region: region, EndpointOverride: endpointOverride, }, @@ -313,7 +313,7 @@ func TestPullImageECRAuthFail(t *testing.T) { authData := &api.RegistryAuthenticationData{ Type: "ecr", ECRAuthData: &api.ECRAuthData{ - RegistryId: registryID, + RegistryID: registryID, Region: region, EndpointOverride: endpointOverride, }, diff --git a/agent/engine/docker_image_manager_integ_test.go b/agent/engine/docker_image_manager_integ_test.go index ad194326b18..1726dd199a8 100644 --- a/agent/engine/docker_image_manager_integ_test.go +++ b/agent/engine/docker_image_manager_integ_test.go @@ -555,7 +555,7 @@ func createImageCleanupHappyTestTask(taskName string) *api.Task { Image: test1Image1Name, Essential: false, DesiredStatus: api.ContainerRunning, - Cpu: 10, + CPU: 10, Memory: 10, }, &api.Container{ @@ -563,7 +563,7 @@ func createImageCleanupHappyTestTask(taskName string) *api.Task { Image: test1Image2Name, Essential: false, DesiredStatus: api.ContainerRunning, - Cpu: 10, + CPU: 10, Memory: 10, }, &api.Container{ @@ -571,7 +571,7 @@ func createImageCleanupHappyTestTask(taskName string) *api.Task { Image: test1Image3Name, Essential: false, DesiredStatus: api.ContainerRunning, - Cpu: 10, + CPU: 10, Memory: 10, }, }, @@ -590,7 +590,7 @@ func createImageCleanupThresholdTestTask(taskName string) *api.Task { Image: test2Image1Name, Essential: false, DesiredStatus: api.ContainerRunning, - Cpu: 10, + CPU: 10, Memory: 10, }, &api.Container{ @@ -598,7 +598,7 @@ func createImageCleanupThresholdTestTask(taskName string) *api.Task { Image: test2Image2Name, Essential: false, DesiredStatus: api.ContainerRunning, - Cpu: 10, + CPU: 10, Memory: 10, }, &api.Container{ @@ -606,7 +606,7 @@ func createImageCleanupThresholdTestTask(taskName string) *api.Task { Image: test2Image3Name, Essential: false, DesiredStatus: api.ContainerRunning, - Cpu: 10, + CPU: 10, Memory: 10, }, }, diff --git a/agent/engine/docker_task_engine.go b/agent/engine/docker_task_engine.go index 9a9f3a339b2..382f3bb296a 100644 --- a/agent/engine/docker_task_engine.go +++ b/agent/engine/docker_task_engine.go @@ -226,26 +226,26 @@ func (engine *DockerTaskEngine) synchronizeState() { continue } for _, cont := range conts { - if cont.DockerId == "" { + if cont.DockerID == "" { log.Debug("Found container potentially created while we were down", "name", cont.DockerName) // Figure out the dockerid describedCont, err := engine.client.InspectContainer(cont.DockerName, inspectContainerTimeout) if err != nil { log.Warn("Could not find matching container for expected", "name", cont.DockerName) } else { - cont.DockerId = describedCont.ID + cont.DockerID = describedCont.ID // update mappings that need dockerid engine.state.AddContainer(cont, task) engine.imageManager.RecordContainerReference(cont.Container) } } - if cont.DockerId != "" { - currentState, metadata := engine.client.DescribeContainer(cont.DockerId) + if cont.DockerID != "" { + currentState, metadata := engine.client.DescribeContainer(cont.DockerID) if metadata.Error != nil { currentState = api.ContainerStopped if !cont.Container.KnownTerminal() { cont.Container.ApplyingError = api.NewNamedError(&ContainerVanishedError{}) - log.Warn("Could not describe previously known container; assuming dead", "err", metadata.Error, "id", cont.DockerId, "name", cont.DockerName) + log.Warn("Could not describe previously known container; assuming dead", "err", metadata.Error, "id", cont.DockerID, "name", cont.DockerName) engine.imageManager.RemoveContainerReferenceFromImageState(cont.Container) } } else { @@ -274,7 +274,7 @@ func (engine *DockerTaskEngine) CheckTaskState(task *api.Task) { if !ok { continue } - status, metadata := engine.client.DescribeContainer(dockerContainer.DockerId) + status, metadata := engine.client.DescribeContainer(dockerContainer.DockerID) engine.processTasks.RLock() managedTask, ok := engine.managedTasks[task.Arn] engine.processTasks.RUnlock() @@ -578,7 +578,7 @@ func (engine *DockerTaskEngine) createContainer(task *api.Task, container *api.C metadata := client.CreateContainer(config, hostConfig, containerName, createContainerTimeout) if metadata.DockerID != "" { - engine.state.AddContainer(&api.DockerContainer{DockerId: metadata.DockerID, DockerName: containerName, Container: container}, task) + engine.state.AddContainer(&api.DockerContainer{DockerID: metadata.DockerID, DockerName: containerName, Container: container}, task) } seelog.Infof("Created docker container for task %s: %s -> %s", task, container, metadata.DockerID) return metadata @@ -604,7 +604,7 @@ func (engine *DockerTaskEngine) startContainer(task *api.Task, container *api.Co Error: CannotStartContainerError{fmt.Errorf("Container not recorded as created")}, } } - return client.StartContainer(dockerContainer.DockerId, startContainerTimeout) + return client.StartContainer(dockerContainer.DockerID, startContainerTimeout) } func (engine *DockerTaskEngine) stopContainer(task *api.Task, container *api.Container) DockerContainerMetadata { @@ -623,7 +623,7 @@ func (engine *DockerTaskEngine) stopContainer(task *api.Task, container *api.Con } } - return engine.client.StopContainer(dockerContainer.DockerId, stopContainerTimeout) + return engine.client.StopContainer(dockerContainer.DockerID, stopContainerTimeout) } func (engine *DockerTaskEngine) removeContainer(task *api.Task, container *api.Container) error { diff --git a/agent/engine/dockerauth/ecr.go b/agent/engine/dockerauth/ecr.go index 1a94b51f0db..a144e99798c 100644 --- a/agent/engine/dockerauth/ecr.go +++ b/agent/engine/dockerauth/ecr.go @@ -52,7 +52,7 @@ func (authProvider *ecrAuthProvider) GetAuthconfig(image string) (docker.AuthCon return docker.AuthConfiguration{}, fmt.Errorf("ecrAuthProvider cannot be used without AuthData") } log.Debugf("Calling ECR.GetAuthorizationToken for %s", image) - authData, err := authProvider.client.GetAuthorizationToken(authProvider.authData.RegistryId) + authData, err := authProvider.client.GetAuthorizationToken(authProvider.authData.RegistryID) if err != nil { return docker.AuthConfiguration{}, err } diff --git a/agent/engine/dockerauth/ecr_test.go b/agent/engine/dockerauth/ecr_test.go index 233ba7fc906..f47c2e7430f 100644 --- a/agent/engine/dockerauth/ecr_test.go +++ b/agent/engine/dockerauth/ecr_test.go @@ -46,7 +46,7 @@ func TestNewAuthProviderECRAuth(t *testing.T) { authData := &api.ECRAuthData{ Region: "us-west-2", - RegistryId: "0123456789012", + RegistryID: "0123456789012", EndpointOverride: "my.endpoint", } @@ -66,7 +66,7 @@ func TestGetAuthConfigSuccess(t *testing.T) { authData := &api.ECRAuthData{ Region: "us-west-2", - RegistryId: "0123456789012", + RegistryID: "0123456789012", EndpointOverride: "my.endpoint", } proxyEndpoint := "proxy" @@ -78,7 +78,7 @@ func TestGetAuthConfigSuccess(t *testing.T) { authData: authData, } - client.EXPECT().GetAuthorizationToken(authData.RegistryId).Return(&ecrapi.AuthorizationData{ + client.EXPECT().GetAuthorizationToken(authData.RegistryID).Return(&ecrapi.AuthorizationData{ ProxyEndpoint: aws.String(proxyEndpointScheme + proxyEndpoint), AuthorizationToken: aws.String(base64.StdEncoding.EncodeToString([]byte(username + ":" + password))), }, nil) @@ -105,7 +105,7 @@ func TestGetAuthConfigNoMatchAuthorizationToken(t *testing.T) { authData := &api.ECRAuthData{ Region: "us-west-2", - RegistryId: "0123456789012", + RegistryID: "0123456789012", EndpointOverride: "my.endpoint", } proxyEndpoint := "proxy" @@ -117,7 +117,7 @@ func TestGetAuthConfigNoMatchAuthorizationToken(t *testing.T) { authData: authData, } - client.EXPECT().GetAuthorizationToken(authData.RegistryId).Return(&ecrapi.AuthorizationData{ + client.EXPECT().GetAuthorizationToken(authData.RegistryID).Return(&ecrapi.AuthorizationData{ ProxyEndpoint: aws.String(proxyEndpointScheme + "notproxy"), AuthorizationToken: aws.String(base64.StdEncoding.EncodeToString([]byte(username + ":" + password))), }, nil) @@ -139,7 +139,7 @@ func TestGetAuthConfigBadBase64(t *testing.T) { authData := &api.ECRAuthData{ Region: "us-west-2", - RegistryId: "0123456789012", + RegistryID: "0123456789012", EndpointOverride: "my.endpoint", } proxyEndpoint := "proxy" @@ -151,7 +151,7 @@ func TestGetAuthConfigBadBase64(t *testing.T) { authData: authData, } - client.EXPECT().GetAuthorizationToken(authData.RegistryId).Return(&ecrapi.AuthorizationData{ + client.EXPECT().GetAuthorizationToken(authData.RegistryID).Return(&ecrapi.AuthorizationData{ ProxyEndpoint: aws.String(proxyEndpointScheme + "notproxy"), AuthorizationToken: aws.String((username + ":" + password)), }, nil) @@ -173,7 +173,7 @@ func TestGetAuthConfigMissingResponse(t *testing.T) { authData := &api.ECRAuthData{ Region: "us-west-2", - RegistryId: "0123456789012", + RegistryID: "0123456789012", EndpointOverride: "my.endpoint", } proxyEndpoint := "proxy" @@ -183,7 +183,7 @@ func TestGetAuthConfigMissingResponse(t *testing.T) { authData: authData, } - client.EXPECT().GetAuthorizationToken(authData.RegistryId) + client.EXPECT().GetAuthorizationToken(authData.RegistryID) authconfig, err := provider.GetAuthconfig(proxyEndpoint + "/myimage") if err == nil { @@ -202,7 +202,7 @@ func TestGetAuthConfigECRError(t *testing.T) { authData := &api.ECRAuthData{ Region: "us-west-2", - RegistryId: "0123456789012", + RegistryID: "0123456789012", EndpointOverride: "my.endpoint", } proxyEndpoint := "proxy" @@ -212,7 +212,7 @@ func TestGetAuthConfigECRError(t *testing.T) { authData: authData, } - client.EXPECT().GetAuthorizationToken(authData.RegistryId).Return(nil, errors.New("test error")) + client.EXPECT().GetAuthorizationToken(authData.RegistryID).Return(nil, errors.New("test error")) authconfig, err := provider.GetAuthconfig(proxyEndpoint + "/myimage") if err == nil { diff --git a/agent/engine/dockerstate/docker_task_engine_state.go b/agent/engine/dockerstate/docker_task_engine_state.go index a1b11850035..f5cfe583fc6 100644 --- a/agent/engine/dockerstate/docker_task_engine_state.go +++ b/agent/engine/dockerstate/docker_task_engine_state.go @@ -125,8 +125,8 @@ func (state *DockerTaskEngineState) RemoveTask(task *api.Task) { delete(state.taskToId, task.Arn) for _, dockerContainer := range containerMap { - delete(state.idToTask, dockerContainer.DockerId) - delete(state.idToContainer, dockerContainer.DockerId) + delete(state.idToTask, dockerContainer.DockerID) + delete(state.idToContainer, dockerContainer.DockerID) } } @@ -163,8 +163,8 @@ func (state *DockerTaskEngineState) AddContainer(container *api.DockerContainer, state.tasks[task.Arn] = task } - if container.DockerId != "" { - state.idToTask[container.DockerId] = task.Arn + if container.DockerID != "" { + state.idToTask[container.DockerID] = task.Arn } existingMap, exists := state.taskToId[task.Arn] if !exists { @@ -173,8 +173,8 @@ func (state *DockerTaskEngineState) AddContainer(container *api.DockerContainer, } existingMap[container.Container.Name] = container - if container.DockerId != "" { - state.idToContainer[container.DockerId] = container + if container.DockerID != "" { + state.idToContainer[container.DockerID] = container } } diff --git a/agent/engine/dockerstate/dockerstate_test.go b/agent/engine/dockerstate/dockerstate_test.go index efcacddd48c..d1c719cd2d2 100644 --- a/agent/engine/dockerstate/dockerstate_test.go +++ b/agent/engine/dockerstate/dockerstate_test.go @@ -97,11 +97,11 @@ func TestTwophaseAddContainer(t *testing.T) { if container.DockerName != "dockerName" { t.Fatal("Incorrect docker name") } - if container.DockerId != "" { + if container.DockerID != "" { t.Fatal("DockerID Should be blank") } - state.AddContainer(&api.DockerContainer{DockerName: "dockerName", Container: testTask.Containers[0], DockerId: "did"}, testTask) + state.AddContainer(&api.DockerContainer{DockerName: "dockerName", Container: testTask.Containers[0], DockerID: "did"}, testTask) containerMap, ok = state.ContainerMapByArn("test") if !ok { @@ -115,7 +115,7 @@ func TestTwophaseAddContainer(t *testing.T) { if container.DockerName != "dockerName" { t.Fatal("Incorrect docker name") } - if container.DockerId != "did" { + if container.DockerID != "did" { t.Fatal("DockerID should have been updated") } @@ -123,7 +123,7 @@ func TestTwophaseAddContainer(t *testing.T) { if !ok { t.Fatal("Could not get container by id") } - if container.DockerName != "dockerName" || container.DockerId != "did" { + if container.DockerName != "dockerName" || container.DockerID != "did" { t.Fatal("Incorrect container fetched") } } @@ -134,7 +134,7 @@ func TestRemoveTask(t *testing.T) { Name: "c1", } testDockerContainer := &api.DockerContainer{ - DockerId: "did", + DockerID: "did", Container: testContainer, } testTask := &api.Task{ diff --git a/agent/engine/dockerstate/testutils/json_test.go b/agent/engine/dockerstate/testutils/json_test.go index 66e8fb72bfa..40d3dad9329 100644 --- a/agent/engine/dockerstate/testutils/json_test.go +++ b/agent/engine/dockerstate/testutils/json_test.go @@ -73,7 +73,7 @@ func TestJsonEncoding(t *testing.T) { testTask := createTestTask("test1", 1) testState.AddTask(testTask) for i, cont := range testTask.Containers { - testState.AddContainer(&api.DockerContainer{DockerId: "docker" + strconv.Itoa(i), DockerName: "someName", Container: cont}, testTask) + testState.AddContainer(&api.DockerContainer{DockerID: "docker" + strconv.Itoa(i), DockerName: "someName", Container: cont}, testTask) } other := decodeEqual(t, testState) _, ok := other.ContainerMapByArn("test1") diff --git a/agent/engine/engine_unix_integ_test.go b/agent/engine/engine_unix_integ_test.go index dbfa08e4e2f..636bdf55558 100644 --- a/agent/engine/engine_unix_integ_test.go +++ b/agent/engine/engine_unix_integ_test.go @@ -62,7 +62,7 @@ func createTestContainer() *api.Container { Command: []string{}, Essential: true, DesiredStatus: api.ContainerRunning, - Cpu: 100, + CPU: 100, Memory: 80, } } @@ -188,7 +188,7 @@ func TestPortForward(t *testing.T) { // Kill the existing container now to make the test run more quickly. containerMap, _ := taskEngine.(*DockerTaskEngine).state.ContainerMapByArn(testTask.Arn) - cid := containerMap[testTask.Containers[0].Name].DockerId + cid := containerMap[testTask.Containers[0].Name].DockerID client, _ := docker.NewClient(endpoint) err = client.KillContainer(docker.KillContainerOptions{ID: cid}) if err != nil { @@ -816,7 +816,7 @@ func TestSignalEvent(t *testing.T) { // Signal the container now containerMap, _ := taskEngine.(*DockerTaskEngine).state.ContainerMapByArn(testTask.Arn) - cid := containerMap[testTask.Containers[0].Name].DockerId + cid := containerMap[testTask.Containers[0].Name].DockerID client, _ := docker.NewClient(endpoint) err := client.KillContainer(docker.KillContainerOptions{ID: cid, Signal: docker.Signal(int(syscall.SIGUSR1))}) if err != nil { diff --git a/agent/engine/engine_windows_integ_test.go b/agent/engine/engine_windows_integ_test.go index 5fd3f9e79bc..193565790d9 100644 --- a/agent/engine/engine_windows_integ_test.go +++ b/agent/engine/engine_windows_integ_test.go @@ -31,7 +31,7 @@ func createTestContainer() *api.Container { Image: "microsoft/windowsservercore:latest", Essential: true, DesiredStatus: api.ContainerRunning, - Cpu: 100, + CPU: 100, Memory: 80, } } diff --git a/agent/handlers/v1_handlers.go b/agent/handlers/v1_handlers.go index 8f328ae9ef7..a74763054b7 100644 --- a/agent/handlers/v1_handlers.go +++ b/agent/handlers/v1_handlers.go @@ -68,7 +68,7 @@ func newTaskResponse(task *api.Task, containerMap map[string]*api.DockerContaine if container.Container.IsInternal { continue } - containers = append(containers, ContainerResponse{container.DockerId, container.DockerName, containerName}) + containers = append(containers, ContainerResponse{container.DockerID, container.DockerName, containerName}) } knownStatus := task.GetKnownStatus() diff --git a/agent/handlers/v1_handlers_test.go b/agent/handlers/v1_handlers_test.go index b051fe8277a..d13962e32ad 100644 --- a/agent/handlers/v1_handlers_test.go +++ b/agent/handlers/v1_handlers_test.go @@ -272,7 +272,7 @@ func stateSetupHelper(state *dockerstate.DockerTaskEngineState, tasks []*api.Tas for _, container := range task.Containers { state.AddContainer(&api.DockerContainer{ Container: container, - DockerId: "dockerid-" + task.Arn + "-" + container.Name, + DockerID: "dockerid-" + task.Arn + "-" + container.Name, DockerName: "dockername-" + task.Arn + "-" + container.Name, }, task) } diff --git a/agent/stats/container_test.go b/agent/stats/container_test.go index 5f672d8e650..28271537547 100644 --- a/agent/stats/container_test.go +++ b/agent/stats/container_test.go @@ -137,7 +137,7 @@ func TestContainerStatsCollectionReconnection(t *testing.T) { close(closedChan) mockContainer := &api.DockerContainer{ - DockerId: dockerID, + DockerID: dockerID, Container: &api.Container{ KnownStatus: api.ContainerRunning, }, @@ -178,7 +178,7 @@ func TestContainerStatsCollectionStopsIfContainerIsTerminal(t *testing.T) { statsErr := fmt.Errorf("test error") mockContainer := &api.DockerContainer{ - DockerId: dockerID, + DockerID: dockerID, Container: &api.Container{ KnownStatus: api.ContainerStopped, }, diff --git a/agent/stats/engine_integ_test.go b/agent/stats/engine_integ_test.go index 8241d88742b..e891d8e10df 100644 --- a/agent/stats/engine_integ_test.go +++ b/agent/stats/engine_integ_test.go @@ -33,7 +33,7 @@ func (resolver *IntegContainerMetadataResolver) addToMap(containerID string) { Version: taskDefinitionVersion, } resolver.containerIDToDockerContainer[containerID] = &api.DockerContainer{ - DockerId: containerID, + DockerID: containerID, Container: &api.Container{}, } } @@ -281,7 +281,7 @@ func TestStatsEngineWithDockerTaskEngine(t *testing.T) { dockerTaskEngine.State().AddTask(&testTask) dockerTaskEngine.State().AddContainer( &api.DockerContainer{ - DockerId: container.ID, + DockerID: container.ID, DockerName: "gremlin", Container: containers[0], }, @@ -405,7 +405,7 @@ func TestStatsEngineWithDockerTaskEngineMissingRemoveEvent(t *testing.T) { dockerTaskEngine.State().AddTask(&testTask) dockerTaskEngine.State().AddContainer( &api.DockerContainer{ - DockerId: container.ID, + DockerID: container.ID, DockerName: "gremlin", Container: containers[0], }, From 0bd64f32c8c7e9d4aeaf84ed91daaea302e20509 Mon Sep 17 00:00:00 2001 From: Samuel Karp Date: Wed, 8 Feb 2017 16:58:44 -0800 Subject: [PATCH 12/27] api: Handle SentStatus more safely --- agent/api/container.go | 16 ++++++++++++ agent/api/task.go | 16 ++++++++++++ agent/api/types.go | 32 +++++++++++------------- agent/engine/docker_task_engine.go | 10 ++++---- agent/eventhandler/handler_test.go | 15 +++++------ agent/eventhandler/task_handler.go | 8 +++--- agent/eventhandler/task_handler_types.go | 4 +-- 7 files changed, 63 insertions(+), 38 deletions(-) diff --git a/agent/api/container.go b/agent/api/container.go index 796fce7ed5f..6907cdcc149 100644 --- a/agent/api/container.go +++ b/agent/api/container.go @@ -62,3 +62,19 @@ func (c *Container) SetDesiredStatus(status ContainerStatus) { c.DesiredStatus = status } + +// GetSentStatus safely returns the SentStatus of the container +func (c *Container) GetSentStatus() ContainerStatus { + c.sentStatusLock.RLock() + defer c.sentStatusLock.RUnlock() + + return c.SentStatus +} + +// SetSentStatus safely sets the SentStatus of the container +func (c *Container) SetSentStatus(status ContainerStatus) { + c.sentStatusLock.Lock() + defer c.sentStatusLock.Unlock() + + c.SentStatus = status +} diff --git a/agent/api/task.go b/agent/api/task.go index 4b727bcb651..b5d93bd0091 100644 --- a/agent/api/task.go +++ b/agent/api/task.go @@ -571,3 +571,19 @@ func (task *Task) SetDesiredStatus(status TaskStatus) { task.DesiredStatus = status } + +// GetSentStatus safely returns the SentStatus of the task +func (task *Task) GetSentStatus() TaskStatus { + task.sentStatusLock.RLock() + defer task.sentStatusLock.RUnlock() + + return task.SentStatus +} + +// SetSentStatus safely sets the SentStatus of the task +func (task *Task) SetSentStatus(status TaskStatus) { + task.sentStatusLock.Lock() + defer task.sentStatusLock.Unlock() + + task.SentStatus = status +} diff --git a/agent/api/types.go b/agent/api/types.go index a4a22f4d545..0a29d8f2924 100644 --- a/agent/api/types.go +++ b/agent/api/types.go @@ -154,7 +154,8 @@ type Task struct { // TODO(samuelkarp) SentStatus needs a lock and setters/getters. // TODO SentStatus should probably be private with appropriately written setter/getter. When this is done, we need // to ensure that the UnmarshalJSON is handled properly so that the state storage continues to work. - SentStatus TaskStatus + SentStatus TaskStatus + sentStatusLock sync.RWMutex StartSequenceNumber int64 StopSequenceNumber int64 @@ -226,11 +227,9 @@ type ContainerStateChange struct { // container ports PortBindings []PortBinding - // This bit is a little hacky; a pointer to the container's sentstatus which - // may be updated to indicate what status was sent. This is used to ensure - // the same event is handled only once. - // TODO(samuelkarp) Change this to expose a *Container and use container.UpdateSentStatus - SentStatus *ContainerStatus + // Container is a pointer to the container involved in the state change that gives the event handler a hook into + // storing what status was sent. This is used to ensure the same event is handled only once. + Container *Container } func (c *ContainerStateChange) String() string { @@ -244,8 +243,8 @@ func (c *ContainerStateChange) String() string { if len(c.PortBindings) != 0 { res += fmt.Sprintf(", Ports %v", c.PortBindings) } - if c.SentStatus != nil { - res += ", Known Sent: " + c.SentStatus.String() + if c.Container != nil { + res += ", Known Sent: " + c.Container.GetSentStatus().String() } return res } @@ -260,18 +259,15 @@ type TaskStateChange struct { // Reason may contain details of why the task stopped Reason string - // As above, this is the same sort of hacky. - // This is a pointer to the task's sent-status that gives the event handler a - // hook into storing metadata about the task on the task such that it follows - // the lifecycle of the task and so on. - // TODO(samuelkarp) Change this to expose a *Task and use task.UpdateSentStatus - SentStatus *TaskStatus + // Task is a pointer to the task involved in the state change that gives the event handler a hook into storing + // what status was sent. This is used to ensure the same event is handled only once. + Task *Task } func (t *TaskStateChange) String() string { res := fmt.Sprintf("%s -> %s", t.TaskArn, t.Status.String()) - if t.SentStatus != nil { - res += ", Known Sent: " + t.SentStatus.String() + if t.Task != nil { + res += ", Known Sent: " + t.Task.GetSentStatus().String() } return res } @@ -344,10 +340,10 @@ type Container struct { ApplyingError *DefaultNamedError // SentStatus represents the last KnownStatus that was sent to the ECS SubmitContainerStateChange API. - // TODO(samuelkarp) SentStatus needs a lock and setters/getters. // TODO SentStatus should probably be private with appropriately written setter/getter. When this is done, we need // to ensure that the UnmarshalJSON is handled properly so that the state storage continues to work. - SentStatus ContainerStatus + SentStatus ContainerStatus + sentStatusLock sync.RWMutex KnownExitCode *int KnownPortBindings []PortBinding diff --git a/agent/engine/docker_task_engine.go b/agent/engine/docker_task_engine.go index 382f3bb296a..d1f8254f6ef 100644 --- a/agent/engine/docker_task_engine.go +++ b/agent/engine/docker_task_engine.go @@ -316,10 +316,10 @@ func (engine *DockerTaskEngine) emitTaskEvent(task *api.Task, reason string) { return } event := api.TaskStateChange{ - TaskArn: task.Arn, - Status: taskKnownStatus, - Reason: reason, - SentStatus: &task.SentStatus, + TaskArn: task.Arn, + Status: taskKnownStatus, + Reason: reason, + Task: task, } log.Info("Task change event", "event", event) engine.taskEvents <- event @@ -374,7 +374,7 @@ func (engine *DockerTaskEngine) emitContainerEvent(task *api.Task, cont *api.Con ExitCode: cont.KnownExitCode, PortBindings: cont.KnownPortBindings, Reason: reason, - SentStatus: &cont.SentStatus, + Container: cont, } log.Debug("Container change event", "event", event) engine.containerEvents <- event diff --git a/agent/eventhandler/handler_test.go b/agent/eventhandler/handler_test.go index debe8c9d685..3a27e0cb625 100644 --- a/agent/eventhandler/handler_test.go +++ b/agent/eventhandler/handler_test.go @@ -28,10 +28,10 @@ import ( ) func contEvent(arn string) api.ContainerStateChange { - return api.ContainerStateChange{TaskArn: arn, ContainerName: "containerName", Status: api.ContainerRunning} + return api.ContainerStateChange{TaskArn: arn, ContainerName: "containerName", Status: api.ContainerRunning, Container: &api.Container{}} } func taskEvent(arn string) api.TaskStateChange { - return api.TaskStateChange{TaskArn: arn, Status: api.TaskRunning} + return api.TaskStateChange{TaskArn: arn, Status: api.TaskRunning, Task: &api.Task{}} } func TestSendsEventsOneContainer(t *testing.T) { @@ -175,20 +175,17 @@ func TestSendsEventsDedupe(t *testing.T) { // Verify that a task doesn't get sent if we already have 'sent' it task1 := taskEvent("alreadySent") - taskRunning := api.TaskRunning - task1.SentStatus = &taskRunning + task1.Task.SetSentStatus(api.TaskRunning) cont1 := contEvent("alreadySent") - containerRunning := api.ContainerRunning - cont1.SentStatus = &containerRunning + cont1.Container.SetSentStatus(api.ContainerRunning) AddContainerEvent(cont1, client) AddTaskEvent(task1, client) task2 := taskEvent("containerSent") - taskNone := api.TaskStatusNone - task2.SentStatus = &taskNone + task2.Task.SetSentStatus(api.TaskStatusNone) cont2 := contEvent("containerSent") - cont2.SentStatus = &containerRunning + cont2.Container.SetSentStatus(api.ContainerRunning) // Expect to send a task status but not a container status called := make(chan struct{}) diff --git a/agent/eventhandler/task_handler.go b/agent/eventhandler/task_handler.go index 28b0da451c7..9ef3fac05bb 100644 --- a/agent/eventhandler/task_handler.go +++ b/agent/eventhandler/task_handler.go @@ -114,8 +114,8 @@ func SubmitTaskEvents(events *eventList, client api.ECSClient) { if err == nil { // submitted; ensure we don't retry it event.containerSent = true - if event.containerChange.SentStatus != nil { - *event.containerChange.SentStatus = event.containerChange.Status + if event.containerChange.Container != nil { + event.containerChange.Container.SetSentStatus(event.containerChange.Status) } statesaver.Save() llog.Debug("Submitted container state change") @@ -130,8 +130,8 @@ func SubmitTaskEvents(events *eventList, client api.ECSClient) { if err == nil { // submitted or can't be retried; ensure we don't retry it event.taskSent = true - if event.taskChange.SentStatus != nil { - *event.taskChange.SentStatus = event.taskChange.Status + if event.taskChange.Task != nil { + event.taskChange.Task.SetSentStatus(event.taskChange.Status) } statesaver.Save() llog.Debug("Submitted task state change") diff --git a/agent/eventhandler/task_handler_types.go b/agent/eventhandler/task_handler_types.go index 2481738575e..f6119c7a688 100644 --- a/agent/eventhandler/task_handler_types.go +++ b/agent/eventhandler/task_handler_types.go @@ -76,7 +76,7 @@ func (event *sendableEvent) taskShouldBeSent() bool { if tevent.Status == api.TaskStatusNone { return false // defensive programming :) } - if event.taskSent || (tevent.SentStatus != nil && *tevent.SentStatus >= tevent.Status) { + if event.taskSent || (tevent.Task != nil && tevent.Task.GetSentStatus() >= tevent.Status) { return false // redundant event } return true @@ -87,7 +87,7 @@ func (event *sendableEvent) containerShouldBeSent() bool { return false } cevent := event.containerChange - if event.containerSent || (cevent.SentStatus != nil && *cevent.SentStatus >= cevent.Status) { + if event.containerSent || (cevent.Container != nil && cevent.Container.GetSentStatus() >= cevent.Status) { return false } return true From 57b21c7ada633c61bf1b11e2918a0e5177940ce9 Mon Sep 17 00:00:00 2001 From: Samuel Karp Date: Thu, 9 Feb 2017 15:26:14 -0800 Subject: [PATCH 13/27] engine: Report task status prior to removal Prior to this change, a race condition exists between reporting task status and task cleanup in the agent. If reporting task status took an excessively long time and ECS_ENGINE_TASK_CLEANUP_WAIT_DURATION is set very short, the containers and associated task metadata could be removed before the ECS backend is informed that the task has stopped. When the task is especially short-lived, it's also possible that the cleanup could occur before the ECS backend is even informed that the task is running. In this situation, it's possible for the ECS backend to re-transmit the task to the agent (assuming that it hadn't started yet) and the de-duplication logic in the agent can break. With this change, we ensure that the task status is reported prior to cleanup actually starting. --- .../engine/docker_image_manager_integ_test.go | 8 +++++++ agent/engine/docker_task_engine_test.go | 2 ++ agent/engine/engine_integ_test.go | 8 ++++++- agent/engine/task_manager.go | 24 +++++++++++++++++++ 4 files changed, 41 insertions(+), 1 deletion(-) diff --git a/agent/engine/docker_image_manager_integ_test.go b/agent/engine/docker_image_manager_integ_test.go index 1726dd199a8..30bbb936f23 100644 --- a/agent/engine/docker_image_manager_integ_test.go +++ b/agent/engine/docker_image_manager_integ_test.go @@ -112,6 +112,7 @@ func TestIntegImageCleanupHappyCase(t *testing.T) { // Verify Task is stopped. verifyTaskIsStopped(taskEvents, testTask) + testTask.SetSentStatus(api.TaskStopped) // Allow Task cleanup to occur time.Sleep(5 * time.Second) @@ -230,6 +231,7 @@ func TestIntegImageCleanupThreshold(t *testing.T) { // Verify Task is stopped verifyTaskIsStopped(taskEvents, testTask) + testTask.SetSentStatus(api.TaskStopped) // Allow Task cleanup to occur time.Sleep(2 * time.Second) @@ -378,6 +380,9 @@ func TestImageWithSameNameAndDifferentID(t *testing.T) { // Verify Task is stopped verifyTaskIsStopped(taskEvents, task1, task2, task3) + task1.SetSentStatus(api.TaskStopped) + task2.SetSentStatus(api.TaskStopped) + task3.SetSentStatus(api.TaskStopped) // Allow Task cleanup to occur time.Sleep(2 * time.Second) @@ -501,6 +506,9 @@ func TestImageWithSameIDAndDifferentNames(t *testing.T) { // Verify Task is stopped verifyTaskIsStopped(taskEvents, task1, task2, task3) + task1.SetSentStatus(api.TaskStopped) + task2.SetSentStatus(api.TaskStopped) + task3.SetSentStatus(api.TaskStopped) // Allow Task cleanup to occur time.Sleep(2 * time.Second) diff --git a/agent/engine/docker_task_engine_test.go b/agent/engine/docker_task_engine_test.go index 2f9693adba4..3ce775a0165 100644 --- a/agent/engine/docker_task_engine_test.go +++ b/agent/engine/docker_task_engine_test.go @@ -187,6 +187,7 @@ func TestBatchContainerHappyPath(t *testing.T) { assert.Equal(t, *cont.ExitCode, 0, "Exit code should be present") } assert.Equal(t, (<-taskEvents).Status, api.TaskStopped, "Task is not in STOPPED state") + sleepTask.SetSentStatus(api.TaskStopped) // Extra events should not block forever; duplicate acs and docker events are possible go func() { eventStream <- createDockerEvent(api.ContainerStopped) }() @@ -317,6 +318,7 @@ func TestRemoveEvents(t *testing.T) { }).Return(nil) taskEngine.AddTask(sleepTaskStop) + sleepTask.SetSentStatus(api.TaskStopped) imageManager.EXPECT().RemoveContainerReferenceFromImageState(gomock.Any()) // trigger cleanup cleanup <- time.Now() diff --git a/agent/engine/engine_integ_test.go b/agent/engine/engine_integ_test.go index e81158d66b8..1e73d4582cf 100644 --- a/agent/engine/engine_integ_test.go +++ b/agent/engine/engine_integ_test.go @@ -41,6 +41,11 @@ const ( credentialsIDIntegTest = "credsid" ) +func init() { + // Set this very low for integ tests only + _stoppedSentWaitInterval = 1 * time.Second +} + func createTestTask(arn string) *api.Task { return &api.Task{ Arn: arn, @@ -183,8 +188,9 @@ func TestSweepContainer(t *testing.T) { defer discardEvents(taskEvents)() // Should be stopped, let's verify it's still listed... - _, ok := taskEngine.(*DockerTaskEngine).State().TaskByArn("testSweepContainer") + task, ok := taskEngine.(*DockerTaskEngine).State().TaskByArn("testSweepContainer") assert.True(t, ok, "Expected task to be present still, but wasn't") + task.SetSentStatus(api.TaskStopped) // cleanupTask waits for TaskStopped to be sent before cleaning time.Sleep(1 * time.Minute) for i := 0; i < 60; i++ { _, ok = taskEngine.(*DockerTaskEngine).State().TaskByArn("testSweepContainer") diff --git a/agent/engine/task_manager.go b/agent/engine/task_manager.go index c829f0c42d6..c9af9050941 100644 --- a/agent/engine/task_manager.go +++ b/agent/engine/task_manager.go @@ -26,6 +26,8 @@ import ( const ( steadyStateTaskVerifyInterval = 10 * time.Minute + stoppedSentWaitInterval = 30 * time.Second + maxStoppedWaitTimes = 72 * time.Hour / stoppedSentWaitInterval ) type acsTaskUpdate struct { @@ -474,6 +476,9 @@ func (mtask *managedTask) time() ttime.Time { return mtask._time } +var _stoppedSentWaitInterval = stoppedSentWaitInterval +var _maxStoppedWaitTimes = int(maxStoppedWaitTimes) + func (mtask *managedTask) cleanupTask(taskStoppedDuration time.Duration) { cleanupTimeDuration := mtask.GetKnownStatusTime().Add(taskStoppedDuration).Sub(ttime.Now()) // There is a potential deadlock here if cleanupTime is negative. Ignore the computed @@ -489,8 +494,27 @@ func (mtask *managedTask) cleanupTask(taskStoppedDuration time.Duration) { cleanupTimeBool <- true close(cleanupTimeBool) }() + // wait for the cleanup time to elapse, signalled by cleanupTimeBool for !mtask.waitEvent(cleanupTimeBool) { } + stoppedSentBool := make(chan bool) + go func() { + for i := 0; i < _maxStoppedWaitTimes; i++ { + // ensure that we block until api.TaskStopped is actually sent + sentStatus := mtask.GetSentStatus() + if sentStatus >= api.TaskStopped { + stoppedSentBool <- true + close(stoppedSentBool) + return + } + seelog.Warnf("Blocking cleanup for task %v until the task has been reported stopped. SentStatus: %v (%d/%d)", mtask, sentStatus, i, _maxStoppedWaitTimes) + mtask._time.Sleep(_stoppedSentWaitInterval) + } + }() + // wait for api.TaskStopped to be sent + for !mtask.waitEvent(stoppedSentBool) { + } + log.Info("Cleaning up task's containers and data", "task", mtask.Task) // For the duration of this, simply discard any task events; this ensures the From eb987248a62396375e2b7b3b80114c75b5212274 Mon Sep 17 00:00:00 2001 From: Samuel Karp Date: Thu, 9 Feb 2017 16:56:30 -0800 Subject: [PATCH 14/27] dockerstate: Create TaskEngineState interface --- agent/agent.go | 2 +- agent/engine/default.go | 2 +- agent/engine/docker_image_manager.go | 4 +- agent/engine/docker_image_manager_test.go | 60 ++--- agent/engine/docker_task_engine.go | 10 +- agent/engine/docker_task_engine_test.go | 2 +- .../dockerstate/docker_task_engine_state.go | 214 +++++++++++------- agent/engine/dockerstate/dockerstate_test.go | 26 +-- agent/engine/dockerstate/generate_mocks.go | 16 ++ agent/engine/dockerstate/json.go | 6 +- .../dockerstate/mocks/dockerstate_mocks.go | 169 ++++++++++++++ .../testutils/docker_state_equal.go | 2 +- .../engine/dockerstate/testutils/json_test.go | 8 +- agent/engine/engine_integ_test.go | 2 +- agent/engine/engine_mocks.go | 2 +- agent/handlers/mocks/handlers_mocks.go | 6 +- agent/handlers/mocks/http/handlers_mocks.go | 2 +- agent/handlers/types.go | 2 +- agent/handlers/v1_handlers.go | 10 +- agent/handlers/v1_handlers_test.go | 6 +- agent/httpclient/mock/httpclient.go | 2 +- agent/logger/audit/mocks/audit_log_mocks.go | 2 +- .../statemanager/mocks/statemanager_mocks.go | 2 +- agent/statemanager/state_manager_test.go | 2 +- agent/statemanager/state_manager_unix_test.go | 4 +- agent/stats/engine.go | 4 +- scripts/generate/mockgen.go | 4 +- 27 files changed, 399 insertions(+), 172 deletions(-) create mode 100644 agent/engine/dockerstate/generate_mocks.go create mode 100644 agent/engine/dockerstate/mocks/dockerstate_mocks.go diff --git a/agent/agent.go b/agent/agent.go index 13b76caad6a..09fd95b3a35 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -115,7 +115,7 @@ func _main() int { // the credentials handler credentialsManager := credentials.NewManager() // Create image manager. This will be used by the task engine for saving image states - state := dockerstate.NewDockerTaskEngineState() + state := dockerstate.NewTaskEngineState() imageManager := engine.NewImageManager(cfg, dockerClient, state) if *versionFlag { versionableEngine := engine.NewTaskEngine(cfg, dockerClient, credentialsManager, containerChangeEventStream, imageManager, state) diff --git a/agent/engine/default.go b/agent/engine/default.go index a6169b43590..fa3f00f87c6 100644 --- a/agent/engine/default.go +++ b/agent/engine/default.go @@ -26,6 +26,6 @@ import ( var log = logger.ForModule("TaskEngine") // NewTaskEngine returns a default TaskEngine -func NewTaskEngine(cfg *config.Config, client DockerClient, credentialsManager credentials.Manager, containerChangeEventStream *eventstream.EventStream, imageManager ImageManager, state *dockerstate.DockerTaskEngineState) TaskEngine { +func NewTaskEngine(cfg *config.Config, client DockerClient, credentialsManager credentials.Manager, containerChangeEventStream *eventstream.EventStream, imageManager ImageManager, state dockerstate.TaskEngineState) TaskEngine { return NewDockerTaskEngine(cfg, client, credentialsManager, containerChangeEventStream, imageManager, state) } diff --git a/agent/engine/docker_image_manager.go b/agent/engine/docker_image_manager.go index 921e73b9957..e9dad1e9969 100644 --- a/agent/engine/docker_image_manager.go +++ b/agent/engine/docker_image_manager.go @@ -50,7 +50,7 @@ type dockerImageManager struct { client DockerClient updateLock sync.RWMutex imageCleanupTicker *time.Ticker - state *dockerstate.DockerTaskEngineState + state dockerstate.TaskEngineState saver statemanager.Saver imageStatesConsideredForDeletion map[string]*image.ImageState minimumAgeBeforeDeletion time.Duration @@ -62,7 +62,7 @@ type dockerImageManager struct { type ImageStatesForDeletion []*image.ImageState // NewImageManager returns a new ImageManager -func NewImageManager(cfg *config.Config, client DockerClient, state *dockerstate.DockerTaskEngineState) ImageManager { +func NewImageManager(cfg *config.Config, client DockerClient, state dockerstate.TaskEngineState) ImageManager { return &dockerImageManager{ client: client, state: state, diff --git a/agent/engine/docker_image_manager_test.go b/agent/engine/docker_image_manager_test.go index 35ff3653bc9..faa5a8f81a6 100644 --- a/agent/engine/docker_image_manager_test.go +++ b/agent/engine/docker_image_manager_test.go @@ -35,7 +35,7 @@ func TestAddAndRemoveContainerToImageStateReferenceHappyPath(t *testing.T) { defer ctrl.Finish() client := NewMockDockerClient(ctrl) - imageManager := NewImageManager(defaultTestConfig(), client, dockerstate.NewDockerTaskEngineState()) + imageManager := NewImageManager(defaultTestConfig(), client, dockerstate.NewTaskEngineState()) imageManager.SetSaver(statemanager.NewNoopStateManager()) container := &api.Container{ @@ -83,7 +83,7 @@ func TestRecordContainerReferenceInspectError(t *testing.T) { imageManager := &dockerImageManager{ client: client, - state: dockerstate.NewDockerTaskEngineState(), + state: dockerstate.NewTaskEngineState(), minimumAgeBeforeDeletion: config.DefaultImageDeletionAge, numImagesToDelete: config.DefaultNumImagesToDeletePerCycle, imageCleanupTimeInterval: config.DefaultImageCleanupTimeInterval, @@ -117,7 +117,7 @@ func TestRecordContainerReferenceWithNoImageName(t *testing.T) { imageManager := &dockerImageManager{ client: client, - state: dockerstate.NewDockerTaskEngineState(), + state: dockerstate.NewTaskEngineState(), minimumAgeBeforeDeletion: config.DefaultImageDeletionAge, numImagesToDelete: config.DefaultNumImagesToDeletePerCycle, imageCleanupTimeInterval: config.DefaultImageCleanupTimeInterval, @@ -160,7 +160,7 @@ func TestAddInvalidContainerReferenceToImageState(t *testing.T) { defer ctrl.Finish() client := NewMockDockerClient(ctrl) - imageManager := NewImageManager(defaultTestConfig(), client, dockerstate.NewDockerTaskEngineState()) + imageManager := NewImageManager(defaultTestConfig(), client, dockerstate.NewTaskEngineState()) imageManager.SetSaver(statemanager.NewNoopStateManager()) container := &api.Container{ @@ -176,7 +176,7 @@ func TestAddContainerReferenceToExistingImageState(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() client := NewMockDockerClient(ctrl) - imageManager := &dockerImageManager{client: client, state: dockerstate.NewDockerTaskEngineState()} + imageManager := &dockerImageManager{client: client, state: dockerstate.NewTaskEngineState()} imageID := "sha256:qwerty" container := &api.Container{ Name: "testContainer", @@ -213,7 +213,7 @@ func TestAddContainerReferenceToExistingImageStateNoState(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() client := NewMockDockerClient(ctrl) - imageManager := &dockerImageManager{client: client, state: dockerstate.NewDockerTaskEngineState()} + imageManager := &dockerImageManager{client: client, state: dockerstate.NewTaskEngineState()} container := &api.Container{ Name: "testContainer", Image: "testContainerImage", @@ -228,7 +228,7 @@ func TestAddContainerReferenceToNewImageState(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() client := NewMockDockerClient(ctrl) - imageManager := &dockerImageManager{client: client, state: dockerstate.NewDockerTaskEngineState()} + imageManager := &dockerImageManager{client: client, state: dockerstate.NewTaskEngineState()} imageID := "sha256:qwerty" var imageSize int64 imageSize = 18767 @@ -248,7 +248,7 @@ func TestAddContainerReferenceToNewImageStateAddedState(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() client := NewMockDockerClient(ctrl) - imageManager := &dockerImageManager{client: client, state: dockerstate.NewDockerTaskEngineState()} + imageManager := &dockerImageManager{client: client, state: dockerstate.NewTaskEngineState()} imageID := "sha256:qwerty" var imageSize int64 imageSize = 18767 @@ -286,7 +286,7 @@ func TestRemoveContainerReferenceFromInvalidImageState(t *testing.T) { defer ctrl.Finish() client := NewMockDockerClient(ctrl) - imageManager := NewImageManager(defaultTestConfig(), client, dockerstate.NewDockerTaskEngineState()) + imageManager := NewImageManager(defaultTestConfig(), client, dockerstate.NewTaskEngineState()) imageManager.SetSaver(statemanager.NewNoopStateManager()) container := &api.Container{ @@ -307,7 +307,7 @@ func TestRemoveInvalidContainerReferenceFromImageState(t *testing.T) { defer ctrl.Finish() client := NewMockDockerClient(ctrl) - imageManager := NewImageManager(defaultTestConfig(), client, dockerstate.NewDockerTaskEngineState()) + imageManager := NewImageManager(defaultTestConfig(), client, dockerstate.NewTaskEngineState()) imageManager.SetSaver(statemanager.NewNoopStateManager()) container := &api.Container{ @@ -324,7 +324,7 @@ func TestRemoveContainerReferenceFromImageStateInspectError(t *testing.T) { defer ctrl.Finish() client := NewMockDockerClient(ctrl) - imageManager := NewImageManager(defaultTestConfig(), client, dockerstate.NewDockerTaskEngineState()) + imageManager := NewImageManager(defaultTestConfig(), client, dockerstate.NewTaskEngineState()) imageManager.SetSaver(statemanager.NewNoopStateManager()) container := &api.Container{ @@ -344,7 +344,7 @@ func TestRemoveContainerReferenceFromImageStateWithNoReference(t *testing.T) { imageManager := &dockerImageManager{ client: client, - state: dockerstate.NewDockerTaskEngineState(), + state: dockerstate.NewTaskEngineState(), minimumAgeBeforeDeletion: config.DefaultImageDeletionAge, numImagesToDelete: config.DefaultNumImagesToDeletePerCycle, imageCleanupTimeInterval: config.DefaultImageCleanupTimeInterval, @@ -380,7 +380,7 @@ func TestGetCandidateImagesForDeletionImageNoImageState(t *testing.T) { imageManager := &dockerImageManager{ client: client, - state: dockerstate.NewDockerTaskEngineState(), + state: dockerstate.NewTaskEngineState(), minimumAgeBeforeDeletion: config.DefaultImageDeletionAge, numImagesToDelete: config.DefaultNumImagesToDeletePerCycle, imageCleanupTimeInterval: config.DefaultImageCleanupTimeInterval, @@ -400,7 +400,7 @@ func TestGetCandidateImagesForDeletionImageJustPulled(t *testing.T) { imageManager := &dockerImageManager{ client: client, - state: dockerstate.NewDockerTaskEngineState(), + state: dockerstate.NewTaskEngineState(), minimumAgeBeforeDeletion: config.DefaultImageDeletionAge, numImagesToDelete: config.DefaultNumImagesToDeletePerCycle, imageCleanupTimeInterval: config.DefaultImageCleanupTimeInterval, @@ -425,7 +425,7 @@ func TestGetCandidateImagesForDeletionImageHasContainerReference(t *testing.T) { imageManager := &dockerImageManager{ client: client, - state: dockerstate.NewDockerTaskEngineState(), + state: dockerstate.NewTaskEngineState(), minimumAgeBeforeDeletion: config.DefaultImageDeletionAge, numImagesToDelete: config.DefaultNumImagesToDeletePerCycle, imageCleanupTimeInterval: config.DefaultImageCleanupTimeInterval, @@ -466,7 +466,7 @@ func TestGetCandidateImagesForDeletionImageHasMoreContainerReferences(t *testing imageManager := &dockerImageManager{ client: client, - state: dockerstate.NewDockerTaskEngineState(), + state: dockerstate.NewTaskEngineState(), minimumAgeBeforeDeletion: config.DefaultImageDeletionAge, numImagesToDelete: config.DefaultNumImagesToDeletePerCycle, imageCleanupTimeInterval: config.DefaultImageCleanupTimeInterval, @@ -519,7 +519,7 @@ func TestGetLeastRecentlyUsedImages(t *testing.T) { defer ctrl.Finish() client := NewMockDockerClient(ctrl) - imageManager := NewImageManager(defaultTestConfig(), client, dockerstate.NewDockerTaskEngineState()) + imageManager := NewImageManager(defaultTestConfig(), client, dockerstate.NewTaskEngineState()) imageStateA := &image.ImageState{ LastUsedAt: time.Now().AddDate(0, -5, 0), @@ -559,7 +559,7 @@ func TestGetLeastRecentlyUsedImagesLessThanFive(t *testing.T) { imageManager := &dockerImageManager{ client: client, - state: dockerstate.NewDockerTaskEngineState(), + state: dockerstate.NewTaskEngineState(), minimumAgeBeforeDeletion: config.DefaultImageDeletionAge, numImagesToDelete: config.DefaultNumImagesToDeletePerCycle, imageCleanupTimeInterval: config.DefaultImageCleanupTimeInterval, @@ -593,7 +593,7 @@ func TestRemoveAlreadyExistingImageNameWithDifferentID(t *testing.T) { imageManager := &dockerImageManager{ client: client, - state: dockerstate.NewDockerTaskEngineState(), + state: dockerstate.NewTaskEngineState(), minimumAgeBeforeDeletion: config.DefaultImageDeletionAge, numImagesToDelete: config.DefaultNumImagesToDeletePerCycle, imageCleanupTimeInterval: config.DefaultImageCleanupTimeInterval, @@ -644,7 +644,7 @@ func TestImageCleanupHappyPath(t *testing.T) { imageManager := &dockerImageManager{ client: client, - state: dockerstate.NewDockerTaskEngineState(), + state: dockerstate.NewTaskEngineState(), minimumAgeBeforeDeletion: 1 * time.Millisecond, numImagesToDelete: config.DefaultNumImagesToDeletePerCycle, imageCleanupTimeInterval: config.DefaultImageCleanupTimeInterval, @@ -696,7 +696,7 @@ func TestImageCleanupCannotRemoveImage(t *testing.T) { imageManager := &dockerImageManager{ client: client, - state: dockerstate.NewDockerTaskEngineState(), + state: dockerstate.NewTaskEngineState(), minimumAgeBeforeDeletion: config.DefaultImageDeletionAge, numImagesToDelete: config.DefaultNumImagesToDeletePerCycle, imageCleanupTimeInterval: config.DefaultImageCleanupTimeInterval, @@ -746,7 +746,7 @@ func TestImageCleanupRemoveImageById(t *testing.T) { imageManager := &dockerImageManager{ client: client, - state: dockerstate.NewDockerTaskEngineState(), + state: dockerstate.NewTaskEngineState(), minimumAgeBeforeDeletion: config.DefaultImageDeletionAge, numImagesToDelete: config.DefaultNumImagesToDeletePerCycle, imageCleanupTimeInterval: config.DefaultImageCleanupTimeInterval, @@ -791,7 +791,7 @@ func TestDeleteImage(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() client := NewMockDockerClient(ctrl) - imageManager := &dockerImageManager{client: client, state: dockerstate.NewDockerTaskEngineState()} + imageManager := &dockerImageManager{client: client, state: dockerstate.NewTaskEngineState()} imageManager.SetSaver(statemanager.NewNoopStateManager()) container := &api.Container{ Name: "testContainer", @@ -820,7 +820,7 @@ func TestDeleteImageNotFoundError(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() client := NewMockDockerClient(ctrl) - imageManager := &dockerImageManager{client: client, state: dockerstate.NewDockerTaskEngineState()} + imageManager := &dockerImageManager{client: client, state: dockerstate.NewTaskEngineState()} imageManager.SetSaver(statemanager.NewNoopStateManager()) container := &api.Container{ Name: "testContainer", @@ -849,7 +849,7 @@ func TestDeleteImageOtherRemoveImageErrors(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() client := NewMockDockerClient(ctrl) - imageManager := &dockerImageManager{client: client, state: dockerstate.NewDockerTaskEngineState()} + imageManager := &dockerImageManager{client: client, state: dockerstate.NewTaskEngineState()} imageManager.SetSaver(statemanager.NewNoopStateManager()) container := &api.Container{ Name: "testContainer", @@ -878,7 +878,7 @@ func TestDeleteImageIDNull(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() client := NewMockDockerClient(ctrl) - imageManager := &dockerImageManager{client: client, state: dockerstate.NewDockerTaskEngineState()} + imageManager := &dockerImageManager{client: client, state: dockerstate.NewTaskEngineState()} imageManager.SetSaver(statemanager.NewNoopStateManager()) imageManager.deleteImage("", nil) } @@ -887,7 +887,7 @@ func TestRemoveLeastRecentlyUsedImageNoImage(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() client := NewMockDockerClient(ctrl) - imageManager := &dockerImageManager{client: client, state: dockerstate.NewDockerTaskEngineState()} + imageManager := &dockerImageManager{client: client, state: dockerstate.NewTaskEngineState()} imageManager.SetSaver(statemanager.NewNoopStateManager()) err := imageManager.removeLeastRecentlyUsedImage() if err == nil { @@ -899,7 +899,7 @@ func TestRemoveUnusedImagesNoImages(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() client := NewMockDockerClient(ctrl) - imageManager := &dockerImageManager{client: client, state: dockerstate.NewDockerTaskEngineState()} + imageManager := &dockerImageManager{client: client, state: dockerstate.NewTaskEngineState()} imageManager.SetSaver(statemanager.NewNoopStateManager()) imageManager.removeUnusedImages() } @@ -908,7 +908,7 @@ func TestGetImageStateFromImageName(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() client := NewMockDockerClient(ctrl) - imageManager := &dockerImageManager{client: client, state: dockerstate.NewDockerTaskEngineState()} + imageManager := &dockerImageManager{client: client, state: dockerstate.NewTaskEngineState()} imageManager.SetSaver(statemanager.NewNoopStateManager()) container := &api.Container{ Name: "testContainer", @@ -932,7 +932,7 @@ func TestGetImageStateFromImageNameNoImageState(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() client := NewMockDockerClient(ctrl) - imageManager := &dockerImageManager{client: client, state: dockerstate.NewDockerTaskEngineState()} + imageManager := &dockerImageManager{client: client, state: dockerstate.NewTaskEngineState()} imageManager.SetSaver(statemanager.NewNoopStateManager()) container := &api.Container{ Name: "testContainer", diff --git a/agent/engine/docker_task_engine.go b/agent/engine/docker_task_engine.go index d1f8254f6ef..9bee45eb740 100644 --- a/agent/engine/docker_task_engine.go +++ b/agent/engine/docker_task_engine.go @@ -62,7 +62,7 @@ type DockerTaskEngine struct { // current state and mappings to/from dockerId and name. // This is used to checkpoint state to disk so tasks may survive agent // failures or updates - state *dockerstate.DockerTaskEngineState + state dockerstate.TaskEngineState managedTasks map[string]*managedTask taskStopGroup *utilsync.SequentialWaitGroup @@ -96,7 +96,7 @@ type DockerTaskEngine struct { // The distinction between created and initialized is that when created it may // be serialized/deserialized, but it will not communicate with docker until it // is also initialized. -func NewDockerTaskEngine(cfg *config.Config, client DockerClient, credentialsManager credentials.Manager, containerChangeEventStream *eventstream.EventStream, imageManager ImageManager, state *dockerstate.DockerTaskEngineState) *DockerTaskEngine { +func NewDockerTaskEngine(cfg *config.Config, client DockerClient, credentialsManager credentials.Manager, containerChangeEventStream *eventstream.EventStream, imageManager ImageManager, state dockerstate.TaskEngineState) *DockerTaskEngine { dockerTaskEngine := &DockerTaskEngine{ cfg: cfg, client: client, @@ -410,8 +410,8 @@ func (engine *DockerTaskEngine) handleDockerEvents(ctx context.Context) { func (engine *DockerTaskEngine) handleDockerEvent(event DockerContainerChangeEvent) bool { log.Debug("Handling a docker event", "event", event) - task, taskFound := engine.state.TaskById(event.DockerID) - cont, containerFound := engine.state.ContainerById(event.DockerID) + task, taskFound := engine.state.TaskByID(event.DockerID) + cont, containerFound := engine.state.ContainerByID(event.DockerID) if !taskFound || !containerFound { log.Debug("Event for container not managed", "dockerId", event.DockerID) return false @@ -717,7 +717,7 @@ func (engine *DockerTaskEngine) transitionContainer(task *api.Task, container *a // State is a function primarily meant for testing usage; it is explicitly not // part of the TaskEngine interface and should not be relied upon. // It returns an internal representation of the state of this DockerTaskEngine. -func (engine *DockerTaskEngine) State() *dockerstate.DockerTaskEngineState { +func (engine *DockerTaskEngine) State() dockerstate.TaskEngineState { return engine.state } diff --git a/agent/engine/docker_task_engine_test.go b/agent/engine/docker_task_engine_test.go index 3ce775a0165..b403e7de9e0 100644 --- a/agent/engine/docker_task_engine_test.go +++ b/agent/engine/docker_task_engine_test.go @@ -53,7 +53,7 @@ func mocks(t *testing.T, cfg *config.Config) (*gomock.Controller, *MockDockerCli containerChangeEventStream := eventstream.NewEventStream("TESTTASKENGINE", context.Background()) containerChangeEventStream.StartListening() imageManager := NewMockImageManager(ctrl) - taskEngine := NewTaskEngine(cfg, client, credentialsManager, containerChangeEventStream, imageManager, dockerstate.NewDockerTaskEngineState()) + taskEngine := NewTaskEngine(cfg, client, credentialsManager, containerChangeEventStream, imageManager, dockerstate.NewTaskEngineState()) taskEngine.(*DockerTaskEngine)._time = mockTime return ctrl, client, mockTime, taskEngine, credentialsManager, imageManager } diff --git a/agent/engine/dockerstate/docker_task_engine_state.go b/agent/engine/dockerstate/docker_task_engine_state.go index f5cfe583fc6..cb24eb07ff0 100644 --- a/agent/engine/dockerstate/docker_task_engine_state.go +++ b/agent/engine/dockerstate/docker_task_engine_state.go @@ -14,6 +14,7 @@ package dockerstate import ( + "encoding/json" "sync" "github.com/aws/amazon-ecs-agent/agent/api" @@ -23,7 +24,36 @@ import ( var log = logger.ForModule("dockerstate") -// dockerTaskEngineState keeps track of all mappings between tasks we know about +// TaskEngineState keeps track of all mappings between tasks we know about +// and containers docker runs +type TaskEngineState interface { + // AllTasks returns all of the tasks + AllTasks() []*api.Task + // AllImageStates returns all of the image.ImageStates + AllImageStates() []*image.ImageState + // ContainerByID returns an api.DockerContainer for a given container ID + ContainerByID(id string) (*api.DockerContainer, bool) + // ContainerMapByArn returns a map of containers belonging to a particular task ARN + ContainerMapByArn(arn string) (map[string]*api.DockerContainer, bool) + // TaskByID returns an api.Task for a given container ID + TaskByID(cid string) (*api.Task, bool) + // TaskByArn returns a task for a given ARN + TaskByArn(arn string) (*api.Task, bool) + // AddTask adds a task to the state to be stored + AddTask(task *api.Task) + // AddContainer adds a container to the state to be stored for a given task + AddContainer(container *api.DockerContainer, task *api.Task) + // AddImageState adds an image.ImageState to be stored + AddImageState(imageState *image.ImageState) + // RemoveTask removes a task from the state + RemoveTask(task *api.Task) + // RemoveImageState removes an image.ImageState + RemoveImageState(imageState *image.ImageState) + json.Marshaler + json.Unmarshaler +} + +// DockerTaskEngineState keeps track of all mappings between tasks we know about // and containers docker runs // It contains a mutex that can be used to ensure out-of-date state cannot be // accessed before an update comes and to ensure multiple goroutines can safely @@ -41,22 +71,62 @@ type DockerTaskEngineState struct { tasks map[string]*api.Task // taskarn -> api.Task idToTask map[string]string // DockerId -> taskarn - taskToId map[string]map[string]*api.DockerContainer // taskarn -> (containername -> api.DockerContainer) + taskToID map[string]map[string]*api.DockerContainer // taskarn -> (containername -> api.DockerContainer) idToContainer map[string]*api.DockerContainer // DockerId -> api.DockerContainer imageStates map[string]*image.ImageState } -func NewDockerTaskEngineState() *DockerTaskEngineState { +// NewTaskEngineState returns a new TaskEngineState +func NewTaskEngineState() TaskEngineState { + return newDockerTaskEngineState() +} + +func newDockerTaskEngineState() *DockerTaskEngineState { return &DockerTaskEngineState{ tasks: make(map[string]*api.Task), idToTask: make(map[string]string), - taskToId: make(map[string]map[string]*api.DockerContainer), + taskToID: make(map[string]map[string]*api.DockerContainer), idToContainer: make(map[string]*api.DockerContainer), imageStates: make(map[string]*image.ImageState), } } -func (state *DockerTaskEngineState) ContainerById(id string) (*api.DockerContainer, bool) { +// AllTasks returns all of the tasks +func (state *DockerTaskEngineState) AllTasks() []*api.Task { + state.lock.RLock() + defer state.lock.RUnlock() + + return state.allTasks() +} + +func (state *DockerTaskEngineState) allTasks() []*api.Task { + ret := make([]*api.Task, len(state.tasks)) + ndx := 0 + for _, task := range state.tasks { + ret[ndx] = task + ndx++ + } + return ret +} + +// AllImageStates returns all of the image.ImageStates +func (state *DockerTaskEngineState) AllImageStates() []*image.ImageState { + state.lock.RLock() + defer state.lock.RUnlock() + + return state.allImageStates() +} + +func (state *DockerTaskEngineState) allImageStates() []*image.ImageState { + var allImageStates []*image.ImageState + for _, imageState := range state.imageStates { + allImageStates = append(allImageStates, imageState) + } + return allImageStates +} + +// ContainerByID returns an api.DockerContainer for a given container ID +func (state *DockerTaskEngineState) ContainerByID(id string) (*api.DockerContainer, bool) { state.lock.RLock() defer state.lock.RUnlock() @@ -64,16 +134,17 @@ func (state *DockerTaskEngineState) ContainerById(id string) (*api.DockerContain return c, ok } +// ContainerMapByArn returns a map of containers belonging to a particular task ARN func (state *DockerTaskEngineState) ContainerMapByArn(arn string) (map[string]*api.DockerContainer, bool) { state.lock.RLock() defer state.lock.RUnlock() - ret, ok := state.taskToId[arn] + ret, ok := state.taskToID[arn] return ret, ok } -// TaskById retrieves the task of a given docker container id -func (state *DockerTaskEngineState) TaskById(cid string) (*api.Task, bool) { +// TaskByID retrieves the task of a given docker container id +func (state *DockerTaskEngineState) TaskByID(cid string) (*api.Task, bool) { state.lock.RLock() defer state.lock.RUnlock() @@ -84,6 +155,19 @@ func (state *DockerTaskEngineState) TaskById(cid string) (*api.Task, bool) { return state.taskByArn(arn) } +// TaskByArn returns a task for a given ARN +func (state *DockerTaskEngineState) TaskByArn(arn string) (*api.Task, bool) { + state.lock.RLock() + defer state.lock.RUnlock() + + return state.taskByArn(arn) +} + +func (state *DockerTaskEngineState) taskByArn(arn string) (*api.Task, bool) { + t, ok := state.tasks[arn] + return t, ok +} + // AddTask adds a new task to the state func (state *DockerTaskEngineState) AddTask(task *api.Task) { state.lock.Lock() @@ -92,6 +176,39 @@ func (state *DockerTaskEngineState) AddTask(task *api.Task) { state.tasks[task.Arn] = task } +// AddContainer adds a container to the state. +// If the container has been added with only a name and no docker-id, this +// updates the state to include the docker id +func (state *DockerTaskEngineState) AddContainer(container *api.DockerContainer, task *api.Task) { + state.lock.Lock() + defer state.lock.Unlock() + if task == nil || container == nil { + log.Crit("Addcontainer called with nil task/container") + return + } + + _, exists := state.tasks[task.Arn] + if !exists { + log.Debug("AddContainer called with unknown task; adding", "arn", task.Arn) + state.tasks[task.Arn] = task + } + + if container.DockerID != "" { + state.idToTask[container.DockerID] = task.Arn + } + existingMap, exists := state.taskToID[task.Arn] + if !exists { + existingMap = make(map[string]*api.DockerContainer, len(task.Containers)) + state.taskToID[task.Arn] = existingMap + } + existingMap[container.Container.Name] = container + + if container.DockerID != "" { + state.idToContainer[container.DockerID] = container + } +} + +// AddImageState adds an image.ImageState to be stored func (state *DockerTaskEngineState) AddImageState(imageState *image.ImageState) { if imageState == nil { log.Debug("Cannot add empty image state") @@ -118,11 +235,11 @@ func (state *DockerTaskEngineState) RemoveTask(task *api.Task) { return } delete(state.tasks, task.Arn) - containerMap, ok := state.taskToId[task.Arn] + containerMap, ok := state.taskToID[task.Arn] if !ok { return } - delete(state.taskToId, task.Arn) + delete(state.taskToID, task.Arn) for _, dockerContainer := range containerMap { delete(state.idToTask, dockerContainer.DockerID) @@ -130,6 +247,7 @@ func (state *DockerTaskEngineState) RemoveTask(task *api.Task) { } } +// RemoveImageState removes an image.ImageState func (state *DockerTaskEngineState) RemoveImageState(imageState *image.ImageState) { if imageState == nil { log.Debug("Cannot remove empty image state") @@ -145,79 +263,3 @@ func (state *DockerTaskEngineState) RemoveImageState(imageState *image.ImageStat } delete(state.imageStates, imageState.Image.ImageID) } - -// AddContainer adds a container to the state. -// If the container has been added with only a name and no docker-id, this -// updates the state to include the docker id -func (state *DockerTaskEngineState) AddContainer(container *api.DockerContainer, task *api.Task) { - state.lock.Lock() - defer state.lock.Unlock() - if task == nil || container == nil { - log.Crit("Addcontainer called with nil task/container") - return - } - - _, exists := state.tasks[task.Arn] - if !exists { - log.Debug("AddContainer called with unknown task; adding", "arn", task.Arn) - state.tasks[task.Arn] = task - } - - if container.DockerID != "" { - state.idToTask[container.DockerID] = task.Arn - } - existingMap, exists := state.taskToId[task.Arn] - if !exists { - existingMap = make(map[string]*api.DockerContainer, len(task.Containers)) - state.taskToId[task.Arn] = existingMap - } - existingMap[container.Container.Name] = container - - if container.DockerID != "" { - state.idToContainer[container.DockerID] = container - } -} - -func (state *DockerTaskEngineState) TaskByArn(arn string) (*api.Task, bool) { - state.lock.RLock() - defer state.lock.RUnlock() - - return state.taskByArn(arn) -} - -func (state *DockerTaskEngineState) taskByArn(arn string) (*api.Task, bool) { - t, ok := state.tasks[arn] - return t, ok -} - -func (state *DockerTaskEngineState) AllTasks() []*api.Task { - state.lock.RLock() - defer state.lock.RUnlock() - - return state.allTasks() -} - -func (state *DockerTaskEngineState) allTasks() []*api.Task { - ret := make([]*api.Task, len(state.tasks)) - ndx := 0 - for _, task := range state.tasks { - ret[ndx] = task - ndx += 1 - } - return ret -} - -func (state *DockerTaskEngineState) AllImageStates() []*image.ImageState { - state.lock.RLock() - defer state.lock.RUnlock() - - return state.allImageStates() -} - -func (state *DockerTaskEngineState) allImageStates() []*image.ImageState { - var allImageStates []*image.ImageState - for _, imageState := range state.imageStates { - allImageStates = append(allImageStates, imageState) - } - return allImageStates -} diff --git a/agent/engine/dockerstate/dockerstate_test.go b/agent/engine/dockerstate/dockerstate_test.go index d1c719cd2d2..581c598bfb2 100644 --- a/agent/engine/dockerstate/dockerstate_test.go +++ b/agent/engine/dockerstate/dockerstate_test.go @@ -22,9 +22,9 @@ import ( ) func TestCreateDockerTaskEngineState(t *testing.T) { - state := NewDockerTaskEngineState() + state := NewTaskEngineState() - if _, ok := state.ContainerById("test"); ok { + if _, ok := state.ContainerByID("test"); ok { t.Error("Empty state should not have a test container") } @@ -32,7 +32,7 @@ func TestCreateDockerTaskEngineState(t *testing.T) { t.Error("Empty state should not have a test task") } - if _, ok := state.TaskById("test"); ok { + if _, ok := state.TaskByID("test"); ok { t.Error("Empty state should not have a test taskid") } @@ -46,7 +46,7 @@ func TestCreateDockerTaskEngineState(t *testing.T) { } func TestAddTask(t *testing.T) { - state := NewDockerTaskEngineState() + state := NewTaskEngineState() testTask := &api.Task{Arn: "test"} state.AddTask(testTask) @@ -65,7 +65,7 @@ func TestAddTask(t *testing.T) { } func TestTwophaseAddContainer(t *testing.T) { - state := NewDockerTaskEngineState() + state := NewTaskEngineState() testTask := &api.Task{Arn: "test", Containers: []*api.Container{&api.Container{ Name: "testContainer", }}} @@ -119,7 +119,7 @@ func TestTwophaseAddContainer(t *testing.T) { t.Fatal("DockerID should have been updated") } - container, ok = state.ContainerById("did") + container, ok = state.ContainerByID("did") if !ok { t.Fatal("Could not get container by id") } @@ -129,7 +129,7 @@ func TestTwophaseAddContainer(t *testing.T) { } func TestRemoveTask(t *testing.T) { - state := NewDockerTaskEngineState() + state := NewTaskEngineState() testContainer := &api.Container{ Name: "c1", } @@ -159,7 +159,7 @@ func TestRemoveTask(t *testing.T) { } func TestAddImageState(t *testing.T) { - state := NewDockerTaskEngineState() + state := NewTaskEngineState() testImage := &image.Image{ImageID: "sha256:imagedigest"} testImageState := &image.ImageState{Image: testImage} @@ -177,7 +177,7 @@ func TestAddImageState(t *testing.T) { } func TestAddEmptyImageState(t *testing.T) { - state := NewDockerTaskEngineState() + state := NewTaskEngineState() state.AddImageState(nil) if len(state.AllImageStates()) != 0 { @@ -186,7 +186,7 @@ func TestAddEmptyImageState(t *testing.T) { } func TestAddEmptyIdImageState(t *testing.T) { - state := NewDockerTaskEngineState() + state := NewTaskEngineState() testImage := &image.Image{ImageID: ""} testImageState := &image.ImageState{Image: testImage} @@ -198,7 +198,7 @@ func TestAddEmptyIdImageState(t *testing.T) { } func TestRemoveImageState(t *testing.T) { - state := NewDockerTaskEngineState() + state := NewTaskEngineState() testImage := &image.Image{ImageID: "sha256:imagedigest"} testImageState := &image.ImageState{Image: testImage} @@ -214,7 +214,7 @@ func TestRemoveImageState(t *testing.T) { } func TestRemoveEmptyImageState(t *testing.T) { - state := NewDockerTaskEngineState() + state := NewTaskEngineState() testImage := &image.Image{ImageID: "sha256:imagedigest"} testImageState := &image.ImageState{Image: testImage} @@ -230,7 +230,7 @@ func TestRemoveEmptyImageState(t *testing.T) { } func TestRemoveNonExistingImageState(t *testing.T) { - state := NewDockerTaskEngineState() + state := NewTaskEngineState() testImage := &image.Image{ImageID: "sha256:imagedigest"} testImageState := &image.ImageState{Image: testImage} diff --git a/agent/engine/dockerstate/generate_mocks.go b/agent/engine/dockerstate/generate_mocks.go new file mode 100644 index 00000000000..c5f0ded6742 --- /dev/null +++ b/agent/engine/dockerstate/generate_mocks.go @@ -0,0 +1,16 @@ +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package dockerstate + +//go:generate go run ../../../scripts/generate/mockgen.go github.com/aws/amazon-ecs-agent/agent/engine/dockerstate TaskEngineState mocks/dockerstate_mocks.go diff --git a/agent/engine/dockerstate/json.go b/agent/engine/dockerstate/json.go index cdb54a93f85..9a618f2f926 100644 --- a/agent/engine/dockerstate/json.go +++ b/agent/engine/dockerstate/json.go @@ -25,8 +25,8 @@ import ( // DockerTaskEngine state type savedState struct { Tasks []*api.Task - IdToContainer map[string]*api.DockerContainer // DockerId -> api.DockerContainer - IdToTask map[string]string // DockerId -> taskarn + IdToContainer map[string]*api.DockerContainer `json:"IdToContainer"` // DockerId -> api.DockerContainer + IdToTask map[string]string `json:"IdToTask"` // DockerId -> taskarn ImageStates []*image.ImageState } @@ -53,7 +53,7 @@ func (state *DockerTaskEngineState) UnmarshalJSON(data []byte) error { // reset it by just creating a new one and swapping shortly. // This also means we don't have to lock for the remainder of this function // because we are the only ones with a reference to clean - clean := NewDockerTaskEngineState() + clean := newDockerTaskEngineState() for _, task := range saved.Tasks { clean.AddTask(task) diff --git a/agent/engine/dockerstate/mocks/dockerstate_mocks.go b/agent/engine/dockerstate/mocks/dockerstate_mocks.go new file mode 100644 index 00000000000..2cde6ebabf4 --- /dev/null +++ b/agent/engine/dockerstate/mocks/dockerstate_mocks.go @@ -0,0 +1,169 @@ +// Copyright 2015-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +// Automatically generated by MockGen. DO NOT EDIT! +// Source: github.com/aws/amazon-ecs-agent/agent/engine/dockerstate (interfaces: TaskEngineState) + +package mock_dockerstate + +import ( + api "github.com/aws/amazon-ecs-agent/agent/api" + image "github.com/aws/amazon-ecs-agent/agent/engine/image" + gomock "github.com/golang/mock/gomock" +) + +// Mock of TaskEngineState interface +type MockTaskEngineState struct { + ctrl *gomock.Controller + recorder *_MockTaskEngineStateRecorder +} + +// Recorder for MockTaskEngineState (not exported) +type _MockTaskEngineStateRecorder struct { + mock *MockTaskEngineState +} + +func NewMockTaskEngineState(ctrl *gomock.Controller) *MockTaskEngineState { + mock := &MockTaskEngineState{ctrl: ctrl} + mock.recorder = &_MockTaskEngineStateRecorder{mock} + return mock +} + +func (_m *MockTaskEngineState) EXPECT() *_MockTaskEngineStateRecorder { + return _m.recorder +} + +func (_m *MockTaskEngineState) AddContainer(_param0 *api.DockerContainer, _param1 *api.Task) { + _m.ctrl.Call(_m, "AddContainer", _param0, _param1) +} + +func (_mr *_MockTaskEngineStateRecorder) AddContainer(arg0, arg1 interface{}) *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "AddContainer", arg0, arg1) +} + +func (_m *MockTaskEngineState) AddImageState(_param0 *image.ImageState) { + _m.ctrl.Call(_m, "AddImageState", _param0) +} + +func (_mr *_MockTaskEngineStateRecorder) AddImageState(arg0 interface{}) *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "AddImageState", arg0) +} + +func (_m *MockTaskEngineState) AddTask(_param0 *api.Task) { + _m.ctrl.Call(_m, "AddTask", _param0) +} + +func (_mr *_MockTaskEngineStateRecorder) AddTask(arg0 interface{}) *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "AddTask", arg0) +} + +func (_m *MockTaskEngineState) AllImageStates() []*image.ImageState { + ret := _m.ctrl.Call(_m, "AllImageStates") + ret0, _ := ret[0].([]*image.ImageState) + return ret0 +} + +func (_mr *_MockTaskEngineStateRecorder) AllImageStates() *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "AllImageStates") +} + +func (_m *MockTaskEngineState) AllTasks() []*api.Task { + ret := _m.ctrl.Call(_m, "AllTasks") + ret0, _ := ret[0].([]*api.Task) + return ret0 +} + +func (_mr *_MockTaskEngineStateRecorder) AllTasks() *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "AllTasks") +} + +func (_m *MockTaskEngineState) ContainerByID(_param0 string) (*api.DockerContainer, bool) { + ret := _m.ctrl.Call(_m, "ContainerByID", _param0) + ret0, _ := ret[0].(*api.DockerContainer) + ret1, _ := ret[1].(bool) + return ret0, ret1 +} + +func (_mr *_MockTaskEngineStateRecorder) ContainerByID(arg0 interface{}) *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "ContainerByID", arg0) +} + +func (_m *MockTaskEngineState) ContainerMapByArn(_param0 string) (map[string]*api.DockerContainer, bool) { + ret := _m.ctrl.Call(_m, "ContainerMapByArn", _param0) + ret0, _ := ret[0].(map[string]*api.DockerContainer) + ret1, _ := ret[1].(bool) + return ret0, ret1 +} + +func (_mr *_MockTaskEngineStateRecorder) ContainerMapByArn(arg0 interface{}) *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "ContainerMapByArn", arg0) +} + +func (_m *MockTaskEngineState) MarshalJSON() ([]byte, error) { + ret := _m.ctrl.Call(_m, "MarshalJSON") + ret0, _ := ret[0].([]byte) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +func (_mr *_MockTaskEngineStateRecorder) MarshalJSON() *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "MarshalJSON") +} + +func (_m *MockTaskEngineState) RemoveImageState(_param0 *image.ImageState) { + _m.ctrl.Call(_m, "RemoveImageState", _param0) +} + +func (_mr *_MockTaskEngineStateRecorder) RemoveImageState(arg0 interface{}) *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "RemoveImageState", arg0) +} + +func (_m *MockTaskEngineState) RemoveTask(_param0 *api.Task) { + _m.ctrl.Call(_m, "RemoveTask", _param0) +} + +func (_mr *_MockTaskEngineStateRecorder) RemoveTask(arg0 interface{}) *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "RemoveTask", arg0) +} + +func (_m *MockTaskEngineState) TaskByArn(_param0 string) (*api.Task, bool) { + ret := _m.ctrl.Call(_m, "TaskByArn", _param0) + ret0, _ := ret[0].(*api.Task) + ret1, _ := ret[1].(bool) + return ret0, ret1 +} + +func (_mr *_MockTaskEngineStateRecorder) TaskByArn(arg0 interface{}) *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "TaskByArn", arg0) +} + +func (_m *MockTaskEngineState) TaskByID(_param0 string) (*api.Task, bool) { + ret := _m.ctrl.Call(_m, "TaskByID", _param0) + ret0, _ := ret[0].(*api.Task) + ret1, _ := ret[1].(bool) + return ret0, ret1 +} + +func (_mr *_MockTaskEngineStateRecorder) TaskByID(arg0 interface{}) *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "TaskByID", arg0) +} + +func (_m *MockTaskEngineState) UnmarshalJSON(_param0 []byte) error { + ret := _m.ctrl.Call(_m, "UnmarshalJSON", _param0) + ret0, _ := ret[0].(error) + return ret0 +} + +func (_mr *_MockTaskEngineStateRecorder) UnmarshalJSON(arg0 interface{}) *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "UnmarshalJSON", arg0) +} diff --git a/agent/engine/dockerstate/testutils/docker_state_equal.go b/agent/engine/dockerstate/testutils/docker_state_equal.go index 33916352bd8..063376c356c 100644 --- a/agent/engine/dockerstate/testutils/docker_state_equal.go +++ b/agent/engine/dockerstate/testutils/docker_state_equal.go @@ -21,7 +21,7 @@ import api_testutils "github.com/aws/amazon-ecs-agent/agent/api/testutils" // DockerStatesEqual determines if the two given dockerstates are equal, for // equal meaning they have the same tasks and their tasks are equal -func DockerStatesEqual(lhs, rhs *dockerstate.DockerTaskEngineState) bool { +func DockerStatesEqual(lhs, rhs dockerstate.TaskEngineState) bool { // Simple equality check; just verify that all tasks are equal lhsTasks := lhs.AllTasks() rhsTasks := rhs.AllTasks() diff --git a/agent/engine/dockerstate/testutils/json_test.go b/agent/engine/dockerstate/testutils/json_test.go index 40d3dad9329..811dd6e4876 100644 --- a/agent/engine/dockerstate/testutils/json_test.go +++ b/agent/engine/dockerstate/testutils/json_test.go @@ -48,12 +48,12 @@ func createTestTask(arn string, numContainers int) *api.Task { return task } -func decodeEqual(t *testing.T, state *dockerstate.DockerTaskEngineState) *dockerstate.DockerTaskEngineState { +func decodeEqual(t *testing.T, state dockerstate.TaskEngineState) dockerstate.TaskEngineState { data, err := json.Marshal(&state) if err != nil { t.Error(err) } - otherState := dockerstate.NewDockerTaskEngineState() + otherState := dockerstate.NewTaskEngineState() err = json.Unmarshal(data, &otherState) if err != nil { t.Error(err) @@ -66,10 +66,10 @@ func decodeEqual(t *testing.T, state *dockerstate.DockerTaskEngineState) *docker } func TestJsonEncoding(t *testing.T) { - state := dockerstate.NewDockerTaskEngineState() + state := dockerstate.NewTaskEngineState() decodeEqual(t, state) - testState := dockerstate.NewDockerTaskEngineState() + testState := dockerstate.NewTaskEngineState() testTask := createTestTask("test1", 1) testState.AddTask(testTask) for i, cont := range testTask.Containers { diff --git a/agent/engine/engine_integ_test.go b/agent/engine/engine_integ_test.go index 1e73d4582cf..bdf095f34ea 100644 --- a/agent/engine/engine_integ_test.go +++ b/agent/engine/engine_integ_test.go @@ -78,7 +78,7 @@ func setup(cfg *config.Config, t *testing.T) (TaskEngine, func(), credentials.Ma t.Fatalf("Error creating Docker client: %v", err) } credentialsManager := credentials.NewManager() - state := dockerstate.NewDockerTaskEngineState() + state := dockerstate.NewTaskEngineState() imageManager := NewImageManager(cfg, dockerClient, state) imageManager.SetSaver(statemanager.NewNoopStateManager()) taskEngine := NewDockerTaskEngine(cfg, dockerClient, credentialsManager, diff --git a/agent/engine/engine_mocks.go b/agent/engine/engine_mocks.go index fdfa2368c14..7e262ae405d 100644 --- a/agent/engine/engine_mocks.go +++ b/agent/engine/engine_mocks.go @@ -1,4 +1,4 @@ -// Copyright 2015-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2015-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/handlers/mocks/handlers_mocks.go b/agent/handlers/mocks/handlers_mocks.go index 76a59507377..b63b06a375e 100644 --- a/agent/handlers/mocks/handlers_mocks.go +++ b/agent/handlers/mocks/handlers_mocks.go @@ -1,4 +1,4 @@ -// Copyright 2015-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2015-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the @@ -42,9 +42,9 @@ func (_m *MockDockerStateResolver) EXPECT() *_MockDockerStateResolverRecorder { return _m.recorder } -func (_m *MockDockerStateResolver) State() *dockerstate.DockerTaskEngineState { +func (_m *MockDockerStateResolver) State() dockerstate.TaskEngineState { ret := _m.ctrl.Call(_m, "State") - ret0, _ := ret[0].(*dockerstate.DockerTaskEngineState) + ret0, _ := ret[0].(dockerstate.TaskEngineState) return ret0 } diff --git a/agent/handlers/mocks/http/handlers_mocks.go b/agent/handlers/mocks/http/handlers_mocks.go index ee13d96ca86..9881a07cacc 100644 --- a/agent/handlers/mocks/http/handlers_mocks.go +++ b/agent/handlers/mocks/http/handlers_mocks.go @@ -1,4 +1,4 @@ -// Copyright 2015-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2015-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/handlers/types.go b/agent/handlers/types.go index 7f9b93e61a7..b8509008d87 100644 --- a/agent/handlers/types.go +++ b/agent/handlers/types.go @@ -41,5 +41,5 @@ type ContainerResponse struct { } type DockerStateResolver interface { - State() *dockerstate.DockerTaskEngineState + State() dockerstate.TaskEngineState } diff --git a/agent/handlers/v1_handlers.go b/agent/handlers/v1_handlers.go index a74763054b7..d6cfcb96bdd 100644 --- a/agent/handlers/v1_handlers.go +++ b/agent/handlers/v1_handlers.go @@ -90,7 +90,7 @@ func newTaskResponse(task *api.Task, containerMap map[string]*api.DockerContaine } } -func newTasksResponse(state *dockerstate.DockerTaskEngineState) *TasksResponse { +func newTasksResponse(state dockerstate.TaskEngineState) *TasksResponse { allTasks := state.AllTasks() taskResponses := make([]*TaskResponse, len(allTasks)) for ndx, task := range allTasks { @@ -102,7 +102,7 @@ func newTasksResponse(state *dockerstate.DockerTaskEngineState) *TasksResponse { } // Creates JSON response and sets the http status code for the task queried. -func createTaskJSONResponse(task *api.Task, found bool, resourceId string, state *dockerstate.DockerTaskEngineState) ([]byte, int) { +func createTaskJSONResponse(task *api.Task, found bool, resourceId string, state dockerstate.TaskEngineState) ([]byte, int) { var responseJSON []byte status := http.StatusOK if found { @@ -134,7 +134,7 @@ func tasksV1RequestHandlerMaker(taskEngine DockerStateResolver) func(http.Respon } if dockerIdExists { // Create TaskResponse for the docker id in the query. - task, found := dockerTaskEngineState.TaskById(dockerId) + task, found := dockerTaskEngineState.TaskByID(dockerId) responseJSON, status = createTaskJSONResponse(task, found, dockerId, dockerTaskEngineState) w.WriteHeader(status) } else if taskArnExists { @@ -161,7 +161,7 @@ func licenseHandler(w http.ResponseWriter, h *http.Request) { } } -func setupServer(containerInstanceArn *string, taskEngine DockerStateResolver, cfg *config.Config) http.Server { +func setupServer(containerInstanceArn *string, taskEngine DockerStateResolver, cfg *config.Config) *http.Server { serverFunctions := map[string]func(w http.ResponseWriter, r *http.Request){ "/v1/metadata": metadataV1RequestHandlerMaker(containerInstanceArn, cfg), "/v1/tasks": tasksV1RequestHandlerMaker(taskEngine), @@ -190,7 +190,7 @@ func setupServer(containerInstanceArn *string, taskEngine DockerStateResolver, c loggingServeMux := http.NewServeMux() loggingServeMux.Handle("/", LoggingHandler{serverMux}) - server := http.Server{ + server := &http.Server{ Addr: ":" + strconv.Itoa(config.AgentIntrospectionPort), Handler: loggingServeMux, ReadTimeout: 5 * time.Second, diff --git a/agent/handlers/v1_handlers_test.go b/agent/handlers/v1_handlers_test.go index d13962e32ad..2180796139a 100644 --- a/agent/handlers/v1_handlers_test.go +++ b/agent/handlers/v1_handlers_test.go @@ -135,7 +135,7 @@ func TestBackendMismatchMapping(t *testing.T) { Containers: containers, } - state := dockerstate.NewDockerTaskEngineState() + state := dockerstate.NewTaskEngineState() stateSetupHelper(state, []*api.Task{testTask}) mockStateResolver.EXPECT().State().Return(state) @@ -266,7 +266,7 @@ var testTasks = []*api.Task{ }, } -func stateSetupHelper(state *dockerstate.DockerTaskEngineState, tasks []*api.Task) { +func stateSetupHelper(state dockerstate.TaskEngineState, tasks []*api.Task) { for _, task := range tasks { state.AddTask(task) for _, container := range task.Containers { @@ -285,7 +285,7 @@ func performMockRequest(t *testing.T, path string) *httptest.ResponseRecorder { mockStateResolver := mock_handlers.NewMockDockerStateResolver(ctrl) - state := dockerstate.NewDockerTaskEngineState() + state := dockerstate.NewTaskEngineState() stateSetupHelper(state, testTasks) mockStateResolver.EXPECT().State().Return(state) diff --git a/agent/httpclient/mock/httpclient.go b/agent/httpclient/mock/httpclient.go index 405c63ea6e1..40af7d40cad 100644 --- a/agent/httpclient/mock/httpclient.go +++ b/agent/httpclient/mock/httpclient.go @@ -1,4 +1,4 @@ -// Copyright 2015-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2015-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/logger/audit/mocks/audit_log_mocks.go b/agent/logger/audit/mocks/audit_log_mocks.go index 41478fe5ece..76faa0aeb01 100644 --- a/agent/logger/audit/mocks/audit_log_mocks.go +++ b/agent/logger/audit/mocks/audit_log_mocks.go @@ -1,4 +1,4 @@ -// Copyright 2015-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2015-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/statemanager/mocks/statemanager_mocks.go b/agent/statemanager/mocks/statemanager_mocks.go index 391a01f93e2..f96d573d787 100644 --- a/agent/statemanager/mocks/statemanager_mocks.go +++ b/agent/statemanager/mocks/statemanager_mocks.go @@ -1,4 +1,4 @@ -// Copyright 2015-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2015-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/statemanager/state_manager_test.go b/agent/statemanager/state_manager_test.go index 3f55650cb9f..b34a7a789e8 100644 --- a/agent/statemanager/state_manager_test.go +++ b/agent/statemanager/state_manager_test.go @@ -40,7 +40,7 @@ func TestLoadsV1DataCorrectly(t *testing.T) { defer cleanup() cfg := &config.Config{DataDir: filepath.Join(".", "testdata", "v1", "1")} - taskEngine := engine.NewTaskEngine(&config.Config{}, nil, nil, nil, nil, dockerstate.NewDockerTaskEngineState()) + taskEngine := engine.NewTaskEngine(&config.Config{}, nil, nil, nil, nil, dockerstate.NewTaskEngineState()) var containerInstanceArn, cluster, savedInstanceID string var sequenceNumber int64 diff --git a/agent/statemanager/state_manager_unix_test.go b/agent/statemanager/state_manager_unix_test.go index 75eaa0f1ca7..5d72bed5c5f 100644 --- a/agent/statemanager/state_manager_unix_test.go +++ b/agent/statemanager/state_manager_unix_test.go @@ -43,7 +43,7 @@ func TestStateManager(t *testing.T) { // Now let's make some state to save containerInstanceArn := "" - taskEngine := engine.NewTaskEngine(&config.Config{}, nil, nil, nil, nil, dockerstate.NewDockerTaskEngineState()) + taskEngine := engine.NewTaskEngine(&config.Config{}, nil, nil, nil, nil, dockerstate.NewTaskEngineState()) manager, err = statemanager.NewStateManager(cfg, statemanager.AddSaveable("TaskEngine", taskEngine), statemanager.AddSaveable("ContainerInstanceArn", &containerInstanceArn)) require.Nil(t, err) @@ -59,7 +59,7 @@ func TestStateManager(t *testing.T) { assertFileMode(t, filepath.Join(tmpDir, "ecs_agent_data.json")) // Now make sure we can load that state sanely - loadedTaskEngine := engine.NewTaskEngine(&config.Config{}, nil, nil, nil, nil, dockerstate.NewDockerTaskEngineState()) + loadedTaskEngine := engine.NewTaskEngine(&config.Config{}, nil, nil, nil, nil, dockerstate.NewTaskEngineState()) var loadedContainerInstanceArn string manager, err = statemanager.NewStateManager(cfg, statemanager.AddSaveable("TaskEngine", &loadedTaskEngine), statemanager.AddSaveable("ContainerInstanceArn", &loadedContainerInstanceArn)) diff --git a/agent/stats/engine.go b/agent/stats/engine.go index d2f73840c5f..443a314e3f5 100644 --- a/agent/stats/engine.go +++ b/agent/stats/engine.go @@ -76,7 +76,7 @@ func (resolver *DockerContainerMetadataResolver) ResolveTask(dockerID string) (* if resolver.dockerTaskEngine == nil { return nil, fmt.Errorf("Docker task engine uninitialized") } - task, found := resolver.dockerTaskEngine.State().TaskById(dockerID) + task, found := resolver.dockerTaskEngine.State().TaskByID(dockerID) if !found { return nil, fmt.Errorf("Could not map docker id to task: %s", dockerID) } @@ -89,7 +89,7 @@ func (resolver *DockerContainerMetadataResolver) ResolveContainer(dockerID strin if resolver.dockerTaskEngine == nil { return nil, fmt.Errorf("Docker task engine uninitialized") } - container, found := resolver.dockerTaskEngine.State().ContainerById(dockerID) + container, found := resolver.dockerTaskEngine.State().ContainerByID(dockerID) if !found { return nil, fmt.Errorf("Could not map docker id to container: %s", dockerID) } diff --git a/scripts/generate/mockgen.go b/scripts/generate/mockgen.go index 34f63694e45..d6bc4b4b491 100644 --- a/scripts/generate/mockgen.go +++ b/scripts/generate/mockgen.go @@ -1,4 +1,4 @@ -// Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2016-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the @@ -55,7 +55,7 @@ func main() { copyrightHeader := fmt.Sprintf(copyrightHeaderFormat, time.Now().Year()) path, _ := filepath.Split(outputPath) - err := os.MkdirAll(path, os.ModeDir) + err := os.MkdirAll(path, os.ModeDir|0755) if err != nil { fmt.Println(err) os.Exit(1) From c9a12956c68fe0b32cfda7852cb33a515536e796 Mon Sep 17 00:00:00 2001 From: Samuel Karp Date: Thu, 9 Feb 2017 18:03:23 -0800 Subject: [PATCH 15/27] engine: Unit tests for managedTask.cleanupTask --- .../engine/docker_image_manager_integ_test.go | 6 +- agent/engine/task_manager.go | 9 +- agent/engine/task_manager_test.go | 179 ++++++++++++++++++ agent/stats/engine_integ_test.go | 4 +- 4 files changed, 192 insertions(+), 6 deletions(-) create mode 100644 agent/engine/task_manager_test.go diff --git a/agent/engine/docker_image_manager_integ_test.go b/agent/engine/docker_image_manager_integ_test.go index 30bbb936f23..3e2df01aa0a 100644 --- a/agent/engine/docker_image_manager_integ_test.go +++ b/agent/engine/docker_image_manager_integ_test.go @@ -234,7 +234,7 @@ func TestIntegImageCleanupThreshold(t *testing.T) { testTask.SetSentStatus(api.TaskStopped) // Allow Task cleanup to occur - time.Sleep(2 * time.Second) + time.Sleep(5 * time.Second) // Verify Task is cleaned up err = verifyTaskIsCleanedUp(taskName, taskEngine) @@ -385,7 +385,7 @@ func TestImageWithSameNameAndDifferentID(t *testing.T) { task3.SetSentStatus(api.TaskStopped) // Allow Task cleanup to occur - time.Sleep(2 * time.Second) + time.Sleep(5 * time.Second) err = verifyTaskIsCleanedUp("task1", taskEngine) assert.NoError(t, err, "task1") @@ -511,7 +511,7 @@ func TestImageWithSameIDAndDifferentNames(t *testing.T) { task3.SetSentStatus(api.TaskStopped) // Allow Task cleanup to occur - time.Sleep(2 * time.Second) + time.Sleep(5 * time.Second) err = verifyTaskIsCleanedUp("task1", taskEngine) assert.NoError(t, err, "task1") diff --git a/agent/engine/task_manager.go b/agent/engine/task_manager.go index c9af9050941..b75d76ca1f5 100644 --- a/agent/engine/task_manager.go +++ b/agent/engine/task_manager.go @@ -498,22 +498,29 @@ func (mtask *managedTask) cleanupTask(taskStoppedDuration time.Duration) { for !mtask.waitEvent(cleanupTimeBool) { } stoppedSentBool := make(chan bool) + taskStopped := false go func() { for i := 0; i < _maxStoppedWaitTimes; i++ { // ensure that we block until api.TaskStopped is actually sent sentStatus := mtask.GetSentStatus() if sentStatus >= api.TaskStopped { stoppedSentBool <- true + taskStopped = true close(stoppedSentBool) return } - seelog.Warnf("Blocking cleanup for task %v until the task has been reported stopped. SentStatus: %v (%d/%d)", mtask, sentStatus, i, _maxStoppedWaitTimes) + seelog.Warnf("Blocking cleanup for task %v until the task has been reported stopped. SentStatus: %v (%d/%d)", mtask, sentStatus, i+1, _maxStoppedWaitTimes) mtask._time.Sleep(_stoppedSentWaitInterval) } + stoppedSentBool <- true }() // wait for api.TaskStopped to be sent for !mtask.waitEvent(stoppedSentBool) { } + if !taskStopped { + seelog.Errorf("Aborting cleanup for task %v as it is not reported stopped. SentStatus: %v", mtask, mtask.GetSentStatus()) + return + } log.Info("Cleaning up task's containers and data", "task", mtask.Task) diff --git a/agent/engine/task_manager_test.go b/agent/engine/task_manager_test.go new file mode 100644 index 00000000000..15b0c999660 --- /dev/null +++ b/agent/engine/task_manager_test.go @@ -0,0 +1,179 @@ +// +build !integration +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package engine + +import ( + "testing" + "time" + + "github.com/aws/amazon-ecs-agent/agent/api" + "github.com/aws/amazon-ecs-agent/agent/engine/dockerstate/mocks" + "github.com/aws/amazon-ecs-agent/agent/engine/testdata" + "github.com/aws/amazon-ecs-agent/agent/statemanager" + "github.com/aws/amazon-ecs-agent/agent/utils/ttime/mocks" + "github.com/stretchr/testify/assert" + + "github.com/golang/mock/gomock" +) + +func TestCleanupTask(t *testing.T) { + ctrl := gomock.NewController(t) + mockTime := mock_ttime.NewMockTime(ctrl) + mockState := mock_dockerstate.NewMockTaskEngineState(ctrl) + mockClient := NewMockDockerClient(ctrl) + mockImageManager := NewMockImageManager(ctrl) + defer ctrl.Finish() + + taskEngine := &DockerTaskEngine{ + saver: statemanager.NewNoopStateManager(), + state: mockState, + client: mockClient, + imageManager: mockImageManager, + } + mTask := &managedTask{ + Task: testdata.LoadTask("sleep5"), + _time: mockTime, + engine: taskEngine, + acsMessages: make(chan acsTransition), + dockerMessages: make(chan dockerContainerChange), + } + mTask.SetKnownStatus(api.TaskStopped) + mTask.SetSentStatus(api.TaskStopped) + container := mTask.Containers[0] + dockerContainer := &api.DockerContainer{ + DockerName: "dockerContainer", + } + + // Expectations for triggering cleanup + now := mTask.GetKnownStatusTime() + taskStoppedDuration := 1 * time.Minute + mockTime.EXPECT().Now().Return(now).AnyTimes() + cleanupTimeTrigger := make(chan time.Time) + mockTime.EXPECT().After(gomock.Any()).Return(cleanupTimeTrigger) + go func() { + cleanupTimeTrigger <- now + }() + + // Expectations to verify that the task gets removed + mockState.EXPECT().ContainerMapByArn(mTask.Arn).Return(map[string]*api.DockerContainer{container.Name: dockerContainer}, true) + mockClient.EXPECT().RemoveContainer(dockerContainer.DockerName, gomock.Any()).Return(nil) + mockImageManager.EXPECT().RemoveContainerReferenceFromImageState(container).Return(nil) + mockState.EXPECT().RemoveTask(mTask.Task) + mTask.cleanupTask(taskStoppedDuration) +} + +func TestCleanupTaskWaitsForStoppedSent(t *testing.T) { + ctrl := gomock.NewController(t) + mockTime := mock_ttime.NewMockTime(ctrl) + mockState := mock_dockerstate.NewMockTaskEngineState(ctrl) + mockClient := NewMockDockerClient(ctrl) + mockImageManager := NewMockImageManager(ctrl) + defer ctrl.Finish() + + taskEngine := &DockerTaskEngine{ + saver: statemanager.NewNoopStateManager(), + state: mockState, + client: mockClient, + imageManager: mockImageManager, + } + mTask := &managedTask{ + Task: testdata.LoadTask("sleep5"), + _time: mockTime, + engine: taskEngine, + acsMessages: make(chan acsTransition), + dockerMessages: make(chan dockerContainerChange), + } + mTask.SetKnownStatus(api.TaskStopped) + mTask.SetSentStatus(api.TaskRunning) + container := mTask.Containers[0] + dockerContainer := &api.DockerContainer{ + DockerName: "dockerContainer", + } + + // Expectations for triggering cleanup + now := mTask.GetKnownStatusTime() + taskStoppedDuration := 1 * time.Minute + mockTime.EXPECT().Now().Return(now).AnyTimes() + cleanupTimeTrigger := make(chan time.Time) + mockTime.EXPECT().After(gomock.Any()).Return(cleanupTimeTrigger) + go func() { + cleanupTimeTrigger <- now + }() + timesCalled := 0 + callsExpected := 3 + mockTime.EXPECT().Sleep(gomock.Any()).AnyTimes().Do(func(_ interface{}) { + timesCalled++ + if timesCalled == callsExpected { + mTask.SetSentStatus(api.TaskStopped) + } else if timesCalled > callsExpected { + t.Errorf("Sleep called too many times, called %d but expected %d", timesCalled, callsExpected) + } + }) + assert.Equal(t, api.TaskRunning, mTask.GetSentStatus()) + + // Expectations to verify that the task gets removed + mockState.EXPECT().ContainerMapByArn(mTask.Arn).Return(map[string]*api.DockerContainer{container.Name: dockerContainer}, true) + mockClient.EXPECT().RemoveContainer(dockerContainer.DockerName, gomock.Any()).Return(nil) + mockImageManager.EXPECT().RemoveContainerReferenceFromImageState(container).Return(nil) + mockState.EXPECT().RemoveTask(mTask.Task) + mTask.cleanupTask(taskStoppedDuration) + assert.Equal(t, api.TaskStopped, mTask.GetSentStatus()) +} + +func TestCleanupTaskGivesUpIfWaitingTooLong(t *testing.T) { + ctrl := gomock.NewController(t) + mockTime := mock_ttime.NewMockTime(ctrl) + mockState := mock_dockerstate.NewMockTaskEngineState(ctrl) + mockClient := NewMockDockerClient(ctrl) + mockImageManager := NewMockImageManager(ctrl) + defer ctrl.Finish() + + taskEngine := &DockerTaskEngine{ + saver: statemanager.NewNoopStateManager(), + state: mockState, + client: mockClient, + imageManager: mockImageManager, + } + mTask := &managedTask{ + Task: testdata.LoadTask("sleep5"), + _time: mockTime, + engine: taskEngine, + acsMessages: make(chan acsTransition), + dockerMessages: make(chan dockerContainerChange), + } + mTask.SetKnownStatus(api.TaskStopped) + mTask.SetSentStatus(api.TaskRunning) + + // Expectations for triggering cleanup + now := mTask.GetKnownStatusTime() + taskStoppedDuration := 1 * time.Minute + mockTime.EXPECT().Now().Return(now).AnyTimes() + cleanupTimeTrigger := make(chan time.Time) + mockTime.EXPECT().After(gomock.Any()).Return(cleanupTimeTrigger) + go func() { + cleanupTimeTrigger <- now + }() + _maxStoppedWaitTimes = 10 + defer func() { + // reset + _maxStoppedWaitTimes = int(maxStoppedWaitTimes) + }() + mockTime.EXPECT().Sleep(gomock.Any()).Times(_maxStoppedWaitTimes) + assert.Equal(t, api.TaskRunning, mTask.GetSentStatus()) + + // No cleanup expected + mTask.cleanupTask(taskStoppedDuration) + assert.Equal(t, api.TaskRunning, mTask.GetSentStatus()) +} diff --git a/agent/stats/engine_integ_test.go b/agent/stats/engine_integ_test.go index e891d8e10df..02c7929bfb6 100644 --- a/agent/stats/engine_integ_test.go +++ b/agent/stats/engine_integ_test.go @@ -246,7 +246,7 @@ func TestStatsEngineWithNewContainers(t *testing.T) { func TestStatsEngineWithDockerTaskEngine(t *testing.T) { containerChangeEventStream := eventStream("TestStatsEngineWithDockerTaskEngine") - taskEngine := ecsengine.NewTaskEngine(&config.Config{}, nil, nil, containerChangeEventStream, nil, dockerstate.NewDockerTaskEngineState()) + taskEngine := ecsengine.NewTaskEngine(&config.Config{}, nil, nil, containerChangeEventStream, nil, dockerstate.NewTaskEngineState()) container, err := createGremlin(client) if err != nil { t.Fatalf("Error creating container: %v", err) @@ -376,7 +376,7 @@ func TestStatsEngineWithDockerTaskEngine(t *testing.T) { func TestStatsEngineWithDockerTaskEngineMissingRemoveEvent(t *testing.T) { containerChangeEventStream := eventStream("TestStatsEngineWithDockerTaskEngine") - taskEngine := ecsengine.NewTaskEngine(&config.Config{}, nil, nil, containerChangeEventStream, nil, dockerstate.NewDockerTaskEngineState()) + taskEngine := ecsengine.NewTaskEngine(&config.Config{}, nil, nil, containerChangeEventStream, nil, dockerstate.NewTaskEngineState()) container, err := createGremlin(client) if err != nil { From 3d154daca403f7a0566164a197b670ca6bbcd12e Mon Sep 17 00:00:00 2001 From: Samuel Karp Date: Thu, 9 Feb 2017 18:11:31 -0800 Subject: [PATCH 16/27] engine: Abort tasks with corrupted internal state --- agent/engine/docker_task_engine.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/agent/engine/docker_task_engine.go b/agent/engine/docker_task_engine.go index 9bee45eb740..c050a708390 100644 --- a/agent/engine/docker_task_engine.go +++ b/agent/engine/docker_task_engine.go @@ -648,11 +648,7 @@ func (engine *DockerTaskEngine) removeContainer(task *api.Task, container *api.C func (engine *DockerTaskEngine) updateTask(task *api.Task, update *api.Task) { managedTask, ok := engine.managedTasks[task.Arn] if !ok { - log.Crit("ACS message for a task we thought we managed, but don't!", "arn", task.Arn) - // Is this the right thing to do? - // Calling startTask should overwrite our bad 'state' data with the new - // task which we do manage.. but this is still scary and shouldn't have happened - engine.startTask(update) + log.Crit("ACS message for a task we thought we managed, but don't! Aborting.", "arn", task.Arn) return } // Keep the lock because sequence numbers cannot be correct unless they are From 4477e305d224709f3718b7960adb93f028d43dc9 Mon Sep 17 00:00:00 2001 From: Samuel Karp Date: Fri, 10 Feb 2017 12:10:40 -0800 Subject: [PATCH 17/27] engine: Extract wait logic for stop reporting --- agent/engine/task_manager.go | 54 +++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/agent/engine/task_manager.go b/agent/engine/task_manager.go index b75d76ca1f5..7167801e0e0 100644 --- a/agent/engine/task_manager.go +++ b/agent/engine/task_manager.go @@ -476,9 +476,6 @@ func (mtask *managedTask) time() ttime.Time { return mtask._time } -var _stoppedSentWaitInterval = stoppedSentWaitInterval -var _maxStoppedWaitTimes = int(maxStoppedWaitTimes) - func (mtask *managedTask) cleanupTask(taskStoppedDuration time.Duration) { cleanupTimeDuration := mtask.GetKnownStatusTime().Add(taskStoppedDuration).Sub(ttime.Now()) // There is a potential deadlock here if cleanupTime is negative. Ignore the computed @@ -497,27 +494,10 @@ func (mtask *managedTask) cleanupTask(taskStoppedDuration time.Duration) { // wait for the cleanup time to elapse, signalled by cleanupTimeBool for !mtask.waitEvent(cleanupTimeBool) { } - stoppedSentBool := make(chan bool) - taskStopped := false - go func() { - for i := 0; i < _maxStoppedWaitTimes; i++ { - // ensure that we block until api.TaskStopped is actually sent - sentStatus := mtask.GetSentStatus() - if sentStatus >= api.TaskStopped { - stoppedSentBool <- true - taskStopped = true - close(stoppedSentBool) - return - } - seelog.Warnf("Blocking cleanup for task %v until the task has been reported stopped. SentStatus: %v (%d/%d)", mtask, sentStatus, i+1, _maxStoppedWaitTimes) - mtask._time.Sleep(_stoppedSentWaitInterval) - } - stoppedSentBool <- true - }() + // wait for api.TaskStopped to be sent - for !mtask.waitEvent(stoppedSentBool) { - } - if !taskStopped { + ok := mtask.waitForStopReported() + if !ok{ seelog.Errorf("Aborting cleanup for task %v as it is not reported stopped. SentStatus: %v", mtask, mtask.GetSentStatus()) return } @@ -573,3 +553,31 @@ func (mtask *managedTask) discardPendingMessages() { } } } + +var _stoppedSentWaitInterval = stoppedSentWaitInterval +var _maxStoppedWaitTimes = int(maxStoppedWaitTimes) + +// waitForStopReported will wait for the task to be reported stopped and return true, or will time-out and return false. +// Messages on the mtask.dockerMessages and mtask.acsMessages channels will be handled while this function is waiting. +func (mtask *managedTask) waitForStopReported() bool { + stoppedSentBool := make(chan bool) + taskStopped := false + go func() { + for i := 0; i < _maxStoppedWaitTimes; i++ { + // ensure that we block until api.TaskStopped is actually sent + sentStatus := mtask.GetSentStatus() + if sentStatus >= api.TaskStopped { + taskStopped = true + break + } + seelog.Warnf("Blocking cleanup for task %v until the task has been reported stopped. SentStatus: %v (%d/%d)", mtask, sentStatus, i+1, _maxStoppedWaitTimes) + mtask._time.Sleep(_stoppedSentWaitInterval) + } + stoppedSentBool <- true + close(stoppedSentBool) + }() + // wait for api.TaskStopped to be sent + for !mtask.waitEvent(stoppedSentBool) { + } + return taskStopped +} \ No newline at end of file From d113ab416fb3b1ae937e5b8f7da02c480e0f8504 Mon Sep 17 00:00:00 2001 From: Samuel Karp Date: Fri, 10 Feb 2017 12:28:27 -0800 Subject: [PATCH 18/27] Update copyright on modified files --- agent/api/container.go | 2 +- agent/api/container_test.go | 2 +- agent/api/port_binding.go | 2 +- agent/api/port_binding_test.go | 2 +- agent/api/task.go | 2 +- agent/api/task_test.go | 2 +- agent/api/testutils/container_equal.go | 2 +- agent/api/testutils/container_equal_test.go | 2 +- agent/api/types.go | 2 +- agent/engine/default.go | 2 +- agent/engine/docker_image_manager.go | 2 +- agent/engine/docker_image_manager_integ_test.go | 2 +- agent/engine/docker_image_manager_test.go | 2 +- agent/engine/dockerauth/ecr.go | 2 +- agent/engine/dockerauth/ecr_test.go | 2 +- agent/engine/dockerstate/docker_task_engine_state.go | 2 +- agent/engine/dockerstate/dockerstate_test.go | 2 +- agent/engine/dockerstate/json.go | 2 +- agent/engine/dockerstate/testutils/docker_state_equal.go | 2 +- agent/engine/dockerstate/testutils/json_test.go | 2 +- agent/engine/engine_unix_integ_test.go | 2 +- agent/engine/engine_windows_integ_test.go | 2 +- agent/eventhandler/handler_test.go | 2 +- agent/eventhandler/task_handler.go | 2 +- agent/eventhandler/task_handler_types.go | 2 +- agent/handlers/types.go | 2 +- agent/handlers/v1_handlers.go | 2 +- agent/handlers/v1_handlers_test.go | 2 +- agent/statemanager/state_manager_test.go | 2 +- agent/statemanager/state_manager_unix_test.go | 2 +- agent/stats/container_test.go | 2 +- agent/stats/engine_integ_test.go | 2 +- 32 files changed, 32 insertions(+), 32 deletions(-) diff --git a/agent/api/container.go b/agent/api/container.go index 6907cdcc149..47102e78d29 100644 --- a/agent/api/container.go +++ b/agent/api/container.go @@ -1,4 +1,4 @@ -// Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/api/container_test.go b/agent/api/container_test.go index 68387fe6c84..f453a937a09 100644 --- a/agent/api/container_test.go +++ b/agent/api/container_test.go @@ -1,4 +1,4 @@ -// Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/api/port_binding.go b/agent/api/port_binding.go index 29da63e115e..66a0e3da192 100644 --- a/agent/api/port_binding.go +++ b/agent/api/port_binding.go @@ -1,4 +1,4 @@ -// Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/api/port_binding_test.go b/agent/api/port_binding_test.go index 6da97b252b8..305477892b0 100644 --- a/agent/api/port_binding_test.go +++ b/agent/api/port_binding_test.go @@ -1,4 +1,4 @@ -// Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/api/task.go b/agent/api/task.go index b5d93bd0091..d420cf530c0 100644 --- a/agent/api/task.go +++ b/agent/api/task.go @@ -1,4 +1,4 @@ -// Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/api/task_test.go b/agent/api/task_test.go index 32c3007a26f..875eccc9810 100644 --- a/agent/api/task_test.go +++ b/agent/api/task_test.go @@ -1,4 +1,4 @@ -// Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/api/testutils/container_equal.go b/agent/api/testutils/container_equal.go index aeea7e3344e..b11ef9f8a13 100644 --- a/agent/api/testutils/container_equal.go +++ b/agent/api/testutils/container_equal.go @@ -1,4 +1,4 @@ -// Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/api/testutils/container_equal_test.go b/agent/api/testutils/container_equal_test.go index dd2cfa690c3..08f065674fe 100644 --- a/agent/api/testutils/container_equal_test.go +++ b/agent/api/testutils/container_equal_test.go @@ -1,4 +1,4 @@ -// Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/api/types.go b/agent/api/types.go index 0a29d8f2924..897ccc77163 100644 --- a/agent/api/types.go +++ b/agent/api/types.go @@ -1,4 +1,4 @@ -// Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/engine/default.go b/agent/engine/default.go index fa3f00f87c6..6dbf4a4b00f 100644 --- a/agent/engine/default.go +++ b/agent/engine/default.go @@ -1,4 +1,4 @@ -// Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/engine/docker_image_manager.go b/agent/engine/docker_image_manager.go index e9dad1e9969..1e572fe25fd 100644 --- a/agent/engine/docker_image_manager.go +++ b/agent/engine/docker_image_manager.go @@ -1,4 +1,4 @@ -// Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/engine/docker_image_manager_integ_test.go b/agent/engine/docker_image_manager_integ_test.go index 3e2df01aa0a..46c7ffc2c2e 100644 --- a/agent/engine/docker_image_manager_integ_test.go +++ b/agent/engine/docker_image_manager_integ_test.go @@ -1,5 +1,5 @@ // +build integration -// Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/engine/docker_image_manager_test.go b/agent/engine/docker_image_manager_test.go index faa5a8f81a6..f05789b7b1c 100644 --- a/agent/engine/docker_image_manager_test.go +++ b/agent/engine/docker_image_manager_test.go @@ -1,5 +1,5 @@ // +build !integration -// Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/engine/dockerauth/ecr.go b/agent/engine/dockerauth/ecr.go index a144e99798c..005a65469a3 100644 --- a/agent/engine/dockerauth/ecr.go +++ b/agent/engine/dockerauth/ecr.go @@ -1,4 +1,4 @@ -// Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/engine/dockerauth/ecr_test.go b/agent/engine/dockerauth/ecr_test.go index f47c2e7430f..9a1d3c0092c 100644 --- a/agent/engine/dockerauth/ecr_test.go +++ b/agent/engine/dockerauth/ecr_test.go @@ -1,5 +1,5 @@ // +build !integration -// Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/engine/dockerstate/docker_task_engine_state.go b/agent/engine/dockerstate/docker_task_engine_state.go index cb24eb07ff0..4fe7a42cc7b 100644 --- a/agent/engine/dockerstate/docker_task_engine_state.go +++ b/agent/engine/dockerstate/docker_task_engine_state.go @@ -1,4 +1,4 @@ -// Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/engine/dockerstate/dockerstate_test.go b/agent/engine/dockerstate/dockerstate_test.go index 581c598bfb2..84f8cbc3830 100644 --- a/agent/engine/dockerstate/dockerstate_test.go +++ b/agent/engine/dockerstate/dockerstate_test.go @@ -1,5 +1,5 @@ // +build !integration -// Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/engine/dockerstate/json.go b/agent/engine/dockerstate/json.go index 9a618f2f926..926e6d87416 100644 --- a/agent/engine/dockerstate/json.go +++ b/agent/engine/dockerstate/json.go @@ -1,4 +1,4 @@ -// Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/engine/dockerstate/testutils/docker_state_equal.go b/agent/engine/dockerstate/testutils/docker_state_equal.go index 063376c356c..e0227dd8105 100644 --- a/agent/engine/dockerstate/testutils/docker_state_equal.go +++ b/agent/engine/dockerstate/testutils/docker_state_equal.go @@ -1,4 +1,4 @@ -// Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/engine/dockerstate/testutils/json_test.go b/agent/engine/dockerstate/testutils/json_test.go index 811dd6e4876..1b3d3838aad 100644 --- a/agent/engine/dockerstate/testutils/json_test.go +++ b/agent/engine/dockerstate/testutils/json_test.go @@ -1,5 +1,5 @@ // +build !integration -// Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/engine/engine_unix_integ_test.go b/agent/engine/engine_unix_integ_test.go index 636bdf55558..eb2ef9f7a85 100644 --- a/agent/engine/engine_unix_integ_test.go +++ b/agent/engine/engine_unix_integ_test.go @@ -1,6 +1,6 @@ // +build !windows,integration -// Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/engine/engine_windows_integ_test.go b/agent/engine/engine_windows_integ_test.go index 193565790d9..382813cab72 100644 --- a/agent/engine/engine_windows_integ_test.go +++ b/agent/engine/engine_windows_integ_test.go @@ -1,6 +1,6 @@ // +build windows,integration -// Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/eventhandler/handler_test.go b/agent/eventhandler/handler_test.go index 3a27e0cb625..818550629b2 100644 --- a/agent/eventhandler/handler_test.go +++ b/agent/eventhandler/handler_test.go @@ -1,4 +1,4 @@ -// Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/eventhandler/task_handler.go b/agent/eventhandler/task_handler.go index 9ef3fac05bb..3b620868c1b 100644 --- a/agent/eventhandler/task_handler.go +++ b/agent/eventhandler/task_handler.go @@ -1,4 +1,4 @@ -// Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/eventhandler/task_handler_types.go b/agent/eventhandler/task_handler_types.go index f6119c7a688..0b915f5c150 100644 --- a/agent/eventhandler/task_handler_types.go +++ b/agent/eventhandler/task_handler_types.go @@ -1,4 +1,4 @@ -// Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/handlers/types.go b/agent/handlers/types.go index b8509008d87..7a8a07f15c8 100644 --- a/agent/handlers/types.go +++ b/agent/handlers/types.go @@ -1,4 +1,4 @@ -// Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/handlers/v1_handlers.go b/agent/handlers/v1_handlers.go index d6cfcb96bdd..0712bc39f34 100644 --- a/agent/handlers/v1_handlers.go +++ b/agent/handlers/v1_handlers.go @@ -1,4 +1,4 @@ -// Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/handlers/v1_handlers_test.go b/agent/handlers/v1_handlers_test.go index 2180796139a..535df1da549 100644 --- a/agent/handlers/v1_handlers_test.go +++ b/agent/handlers/v1_handlers_test.go @@ -1,4 +1,4 @@ -// Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/statemanager/state_manager_test.go b/agent/statemanager/state_manager_test.go index b34a7a789e8..28e27006f9d 100644 --- a/agent/statemanager/state_manager_test.go +++ b/agent/statemanager/state_manager_test.go @@ -1,4 +1,4 @@ -// Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/statemanager/state_manager_unix_test.go b/agent/statemanager/state_manager_unix_test.go index 5d72bed5c5f..13f1be2eac1 100644 --- a/agent/statemanager/state_manager_unix_test.go +++ b/agent/statemanager/state_manager_unix_test.go @@ -1,6 +1,6 @@ // +build !windows -// Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/stats/container_test.go b/agent/stats/container_test.go index 28271537547..5b092ff3830 100644 --- a/agent/stats/container_test.go +++ b/agent/stats/container_test.go @@ -1,5 +1,5 @@ //+build !integration -// Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/stats/engine_integ_test.go b/agent/stats/engine_integ_test.go index 02c7929bfb6..ca4fc0c8dc6 100644 --- a/agent/stats/engine_integ_test.go +++ b/agent/stats/engine_integ_test.go @@ -1,6 +1,6 @@ //+build !windows,integration // Disabled on Windows until Stats are actually supported -// Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the From 61371b469c0bb5eb7bda07684e942dbfbc9d2d2b Mon Sep 17 00:00:00 2001 From: Samuel Karp Date: Fri, 10 Feb 2017 15:25:49 -0800 Subject: [PATCH 19/27] engine: Simplify code in mtask.cleanupTask --- agent/engine/dockerstate/docker_task_engine_state.go | 2 +- agent/engine/task_manager.go | 10 +++------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/agent/engine/dockerstate/docker_task_engine_state.go b/agent/engine/dockerstate/docker_task_engine_state.go index 4fe7a42cc7b..199289aed75 100644 --- a/agent/engine/dockerstate/docker_task_engine_state.go +++ b/agent/engine/dockerstate/docker_task_engine_state.go @@ -225,7 +225,7 @@ func (state *DockerTaskEngineState) AddImageState(imageState *image.ImageState) } // RemoveTask removes a task from this state. It removes all containers and -// other associated metadata. It does aquire the write lock. +// other associated metadata. It does acquire the write lock. func (state *DockerTaskEngineState) RemoveTask(task *api.Task) { state.lock.Lock() defer state.lock.Unlock() diff --git a/agent/engine/task_manager.go b/agent/engine/task_manager.go index 7167801e0e0..f3e5287c136 100644 --- a/agent/engine/task_manager.go +++ b/agent/engine/task_manager.go @@ -507,16 +507,12 @@ func (mtask *managedTask) cleanupTask(taskStoppedDuration time.Duration) { // For the duration of this, simply discard any task events; this ensures the // speedy processing of other events for other tasks handleCleanupDone := make(chan struct{}) - go func() { - mtask.engine.sweepTask(mtask.Task) - mtask.engine.state.RemoveTask(mtask.Task) - handleCleanupDone <- struct{}{} - }() // discard events while the task is being removed from engine state - mtask.discardEventsUntil(handleCleanupDone) + go mtask.discardEventsUntil(handleCleanupDone) + mtask.engine.sweepTask(mtask.Task) + mtask.engine.state.RemoveTask(mtask.Task) log.Debug("Finished removing task data; removing from state no longer managing", "task", mtask.Task) // Now remove ourselves from the global state and cleanup channels - go mtask.discardEventsUntil(handleCleanupDone) // keep discarding events until the task is fully gone mtask.engine.processTasks.Lock() delete(mtask.engine.managedTasks, mtask.Arn) handleCleanupDone <- struct{}{} From 38165f21314555be6fb1e048ca562e8f0442f9d1 Mon Sep 17 00:00:00 2001 From: Vinothkumar Siddharth Date: Wed, 1 Feb 2017 11:12:44 -0800 Subject: [PATCH 20/27] Update LICENSE to 2017 Fixes #686 Signed-off-by: Vinothkumar Siddharth --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index dde93d9b75f..309e3816c3e 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of From 017dcbefed7e7725b9b04c8d971c50c8f4b94e35 Mon Sep 17 00:00:00 2001 From: Vinothkumar Siddharth Date: Thu, 2 Feb 2017 20:10:13 +0000 Subject: [PATCH 21/27] Update gogenerate mock interfaces Signed-off-by: Vinothkumar Siddharth --- .../acs/update_handler/os/mock/filesystem.go | 2 +- agent/api/mocks/api_mocks.go | 2 +- agent/async/mocks/async_mocks.go | 2 +- agent/credentials/mocks/credentials_mocks.go | 2 +- agent/ec2/mocks/ec2_mocks.go | 2 +- agent/ecr/mocks/ecr_mocks.go | 20 +++++++++---------- .../dockerclient/mocks/dockerclient_mocks.go | 2 +- agent/stats/mock/engine.go | 2 +- agent/stats/resolver/mock/resolver.go | 2 +- agent/utils/mocks/utils_mocks.go | 2 +- agent/utils/ttime/mocks/time_mocks.go | 2 +- 11 files changed, 20 insertions(+), 20 deletions(-) diff --git a/agent/acs/update_handler/os/mock/filesystem.go b/agent/acs/update_handler/os/mock/filesystem.go index eb2537d7b32..e3041e74628 100644 --- a/agent/acs/update_handler/os/mock/filesystem.go +++ b/agent/acs/update_handler/os/mock/filesystem.go @@ -1,4 +1,4 @@ -// Copyright 2015-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2015-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/api/mocks/api_mocks.go b/agent/api/mocks/api_mocks.go index 8f45103da97..4175ba6211b 100644 --- a/agent/api/mocks/api_mocks.go +++ b/agent/api/mocks/api_mocks.go @@ -1,4 +1,4 @@ -// Copyright 2015-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2015-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/async/mocks/async_mocks.go b/agent/async/mocks/async_mocks.go index 537b9dd3fbd..157f1111dae 100644 --- a/agent/async/mocks/async_mocks.go +++ b/agent/async/mocks/async_mocks.go @@ -1,4 +1,4 @@ -// Copyright 2015-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2015-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/credentials/mocks/credentials_mocks.go b/agent/credentials/mocks/credentials_mocks.go index 561bb9a002a..84bcbbd5f4b 100644 --- a/agent/credentials/mocks/credentials_mocks.go +++ b/agent/credentials/mocks/credentials_mocks.go @@ -1,4 +1,4 @@ -// Copyright 2015-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2015-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/ec2/mocks/ec2_mocks.go b/agent/ec2/mocks/ec2_mocks.go index c06a8dcb328..d20716393cf 100644 --- a/agent/ec2/mocks/ec2_mocks.go +++ b/agent/ec2/mocks/ec2_mocks.go @@ -1,4 +1,4 @@ -// Copyright 2015-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2015-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/ecr/mocks/ecr_mocks.go b/agent/ecr/mocks/ecr_mocks.go index f88467386c1..a79a9c55788 100644 --- a/agent/ecr/mocks/ecr_mocks.go +++ b/agent/ecr/mocks/ecr_mocks.go @@ -1,4 +1,4 @@ -// Copyright 2015-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2015-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the @@ -17,8 +17,8 @@ package mock_ecr import ( - ecr "github.com/aws/amazon-ecs-agent/agent/ecr" - ecr0 "github.com/aws/amazon-ecs-agent/agent/ecr/model/ecr" + ecr0 "github.com/aws/amazon-ecs-agent/agent/ecr" + ecr "github.com/aws/amazon-ecs-agent/agent/ecr/model/ecr" gomock "github.com/golang/mock/gomock" ) @@ -43,9 +43,9 @@ func (_m *MockECRSDK) EXPECT() *_MockECRSDKRecorder { return _m.recorder } -func (_m *MockECRSDK) GetAuthorizationToken(_param0 *ecr0.GetAuthorizationTokenInput) (*ecr0.GetAuthorizationTokenOutput, error) { +func (_m *MockECRSDK) GetAuthorizationToken(_param0 *ecr.GetAuthorizationTokenInput) (*ecr.GetAuthorizationTokenOutput, error) { ret := _m.ctrl.Call(_m, "GetAuthorizationToken", _param0) - ret0, _ := ret[0].(*ecr0.GetAuthorizationTokenOutput) + ret0, _ := ret[0].(*ecr.GetAuthorizationTokenOutput) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -75,9 +75,9 @@ func (_m *MockECRFactory) EXPECT() *_MockECRFactoryRecorder { return _m.recorder } -func (_m *MockECRFactory) GetClient(_param0 string, _param1 string) ecr.ECRClient { +func (_m *MockECRFactory) GetClient(_param0 string, _param1 string) ecr0.ECRClient { ret := _m.ctrl.Call(_m, "GetClient", _param0, _param1) - ret0, _ := ret[0].(ecr.ECRClient) + ret0, _ := ret[0].(ecr0.ECRClient) return ret0 } @@ -106,9 +106,9 @@ func (_m *MockECRClient) EXPECT() *_MockECRClientRecorder { return _m.recorder } -func (_m *MockECRClient) GetAuthorizationToken(_param0 string) (*ecr0.AuthorizationData, error) { +func (_m *MockECRClient) GetAuthorizationToken(_param0 string) (*ecr.AuthorizationData, error) { ret := _m.ctrl.Call(_m, "GetAuthorizationToken", _param0) - ret0, _ := ret[0].(*ecr0.AuthorizationData) + ret0, _ := ret[0].(*ecr.AuthorizationData) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -117,7 +117,7 @@ func (_mr *_MockECRClientRecorder) GetAuthorizationToken(arg0 interface{}) *gomo return _mr.mock.ctrl.RecordCall(_mr.mock, "GetAuthorizationToken", arg0) } -func (_m *MockECRClient) IsTokenValid(_param0 *ecr0.AuthorizationData) bool { +func (_m *MockECRClient) IsTokenValid(_param0 *ecr.AuthorizationData) bool { ret := _m.ctrl.Call(_m, "IsTokenValid", _param0) ret0, _ := ret[0].(bool) return ret0 diff --git a/agent/engine/dockerclient/mocks/dockerclient_mocks.go b/agent/engine/dockerclient/mocks/dockerclient_mocks.go index fd86437733a..de88eed766d 100644 --- a/agent/engine/dockerclient/mocks/dockerclient_mocks.go +++ b/agent/engine/dockerclient/mocks/dockerclient_mocks.go @@ -1,4 +1,4 @@ -// Copyright 2015-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2015-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/stats/mock/engine.go b/agent/stats/mock/engine.go index a138d39a8c3..c6b64a110b2 100644 --- a/agent/stats/mock/engine.go +++ b/agent/stats/mock/engine.go @@ -1,4 +1,4 @@ -// Copyright 2015-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2015-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/stats/resolver/mock/resolver.go b/agent/stats/resolver/mock/resolver.go index 6f0e2b53619..74ee5c3c46e 100644 --- a/agent/stats/resolver/mock/resolver.go +++ b/agent/stats/resolver/mock/resolver.go @@ -1,4 +1,4 @@ -// Copyright 2015-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2015-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/utils/mocks/utils_mocks.go b/agent/utils/mocks/utils_mocks.go index 50de79ef552..1d7b8b26bdf 100644 --- a/agent/utils/mocks/utils_mocks.go +++ b/agent/utils/mocks/utils_mocks.go @@ -1,4 +1,4 @@ -// Copyright 2015-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2015-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the diff --git a/agent/utils/ttime/mocks/time_mocks.go b/agent/utils/ttime/mocks/time_mocks.go index 533eb1c608a..3a2b1ae88e5 100644 --- a/agent/utils/ttime/mocks/time_mocks.go +++ b/agent/utils/ttime/mocks/time_mocks.go @@ -1,4 +1,4 @@ -// Copyright 2015-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2015-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the From 43ed35a7f75a9e7a447b7533c079a2eece8d748d Mon Sep 17 00:00:00 2001 From: Vinothkumar Siddharth Date: Mon, 6 Feb 2017 19:52:07 +0000 Subject: [PATCH 22/27] Update README for `ECS_INSTANCE_ATTRIBUTES` Fixes #698 Signed-off-by: Vinothkumar Siddharth --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e7416665e47..fb59140c258 100644 --- a/README.md +++ b/README.md @@ -174,7 +174,7 @@ configure them as something other than the defaults. | `ECS_IMAGE_CLEANUP_INTERVAL` | 30m | The time interval between automated image cleanup cycles. If set to less than 10 minutes, the value is ignored. | 30m | 30m | | `ECS_IMAGE_MINIMUM_CLEANUP_AGE` | 30m | The minimum time interval between when an image is pulled and when it can be considered for automated image cleanup. | 1h | 1h | | `ECS_NUM_IMAGES_DELETE_PER_CYCLE` | 5 | The maximum number of images to delete in a single automated image cleanup cycle. If set to less than 1, the value is ignored. | 5 | 5 | -| `ECS_INSTANCE_ATTRIBUTES` | `[{"stack": "prod"}]` | A list of custom attributes, in JSON form, to apply to your container instances. | `[]` | `[]` | +| `ECS_INSTANCE_ATTRIBUTES` | `{"stack": "prod"}` | A list of custom attributes, in JSON form, to apply to your container instances. | `{}` | `{}` | ### Persistence From 2eda892f61b1a94c1b8f44e2fe31ec95913ef862 Mon Sep 17 00:00:00 2001 From: Vinothkumar Siddharth Date: Mon, 13 Feb 2017 13:59:31 -0800 Subject: [PATCH 23/27] Update wording from `list` to `map` for clarity Signed-off-by: Vinothkumar Siddharth --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fb59140c258..406c5e8b197 100644 --- a/README.md +++ b/README.md @@ -174,7 +174,7 @@ configure them as something other than the defaults. | `ECS_IMAGE_CLEANUP_INTERVAL` | 30m | The time interval between automated image cleanup cycles. If set to less than 10 minutes, the value is ignored. | 30m | 30m | | `ECS_IMAGE_MINIMUM_CLEANUP_AGE` | 30m | The minimum time interval between when an image is pulled and when it can be considered for automated image cleanup. | 1h | 1h | | `ECS_NUM_IMAGES_DELETE_PER_CYCLE` | 5 | The maximum number of images to delete in a single automated image cleanup cycle. If set to less than 1, the value is ignored. | 5 | 5 | -| `ECS_INSTANCE_ATTRIBUTES` | `{"stack": "prod"}` | A list of custom attributes, in JSON form, to apply to your container instances. | `{}` | `{}` | +| `ECS_INSTANCE_ATTRIBUTES` | `{"stack": "prod"}` | A map of custom attributes, in JSON form, to apply to your container instances. | `{}` | `{}` | ### Persistence From 0e6ebdadfff57167188f599b82e2f0d2f0f31d44 Mon Sep 17 00:00:00 2001 From: Vinothkumar Siddharth Date: Wed, 8 Feb 2017 13:28:01 -0800 Subject: [PATCH 24/27] Adding functional tests for Custom-Instance-Attributes * This patch tests the happy paths of the custom-instance-attribute feature. The `ECS_INSTANCE_ATTRIBUTE` holds a set of key=value pairs which are exposed via the DescribeContainerInstances API. Each key or value can contain upto 128 characters and we currently only support a maximum of 10 custom attributes per instance. http://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_PutAttributes.html describes more details about the limits for the attributes. Signed-off-by: Vinothkumar Siddharth --- .../tests/functionaltests_test.go | 47 +++++++++++++++++++ .../tests/functionaltests_unix_test.go | 8 ++-- agent/functional_tests/util/utils.go | 9 ++++ agent/functional_tests/util/utils_unix.go | 5 ++ agent/functional_tests/util/utils_windows.go | 8 +++- 5 files changed, 71 insertions(+), 6 deletions(-) diff --git a/agent/functional_tests/tests/functionaltests_test.go b/agent/functional_tests/tests/functionaltests_test.go index 681b466886c..812c46315ac 100644 --- a/agent/functional_tests/tests/functionaltests_test.go +++ b/agent/functional_tests/tests/functionaltests_test.go @@ -19,10 +19,15 @@ import ( "fmt" "os" "reflect" + "strconv" "testing" "time" + ecsapi "github.com/aws/amazon-ecs-agent/agent/ecs_client/model/ecs" . "github.com/aws/amazon-ecs-agent/agent/functional_tests/util" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) const ( @@ -270,3 +275,45 @@ func networkModeTest(t *testing.T, agent *TestAgent, mode string) error { } return nil } + +// TestCustomAttributesWithMaxOptions tests the ECS_INSTANCE_ATTRIBUTES +// upon agent registration with maximum number of supported key, value pairs +func TestCustomAttributesWithMaxOptions(t *testing.T) { + maxAttributes := 10 + customAttributes := `{ + "key1": "val1", + "key2": "val2", + "key3": "val3", + "key4": "val4", + "key5": "val5", + "key6": "val6", + "key7": "val7", + "key8": "val8", + "key9": "val9", + "key0": "val0" + }` + os.Setenv("ECS_INSTANCE_ATTRIBUTES", customAttributes) + defer os.Unsetenv("ECS_INSTANCE_ATTRIBUTES") + + agent := RunAgent(t, nil) + defer agent.Cleanup() + + params := &ecsapi.DescribeContainerInstancesInput{ + Cluster: &agent.Cluster, + ContainerInstances: []*string{&agent.ContainerInstanceArn}, + } + + resp, err := ECS.DescribeContainerInstances(params) + require.NoError(t, err) + require.NotEmpty(t, resp.ContainerInstances) + require.Len(t, resp.ContainerInstances, 1) + + attribMap := AttributesToMap(resp.ContainerInstances[0].Attributes) + assert.NotEmpty(t, attribMap) + + for i := 0; i < maxAttributes; i++ { + k := "key" + strconv.Itoa(i) + v := "val" + strconv.Itoa(i) + assert.Equal(t, v, attribMap[k], "Values should match") + } +} diff --git a/agent/functional_tests/tests/functionaltests_unix_test.go b/agent/functional_tests/tests/functionaltests_unix_test.go index 1b8959a6e3e..79e2c1a3d21 100644 --- a/agent/functional_tests/tests/functionaltests_unix_test.go +++ b/agent/functional_tests/tests/functionaltests_unix_test.go @@ -27,7 +27,7 @@ import ( "testing" "time" - "github.com/aws/amazon-ecs-agent/agent/ecs_client/model/ecs" + ecsapi "github.com/aws/amazon-ecs-agent/agent/ecs_client/model/ecs" . "github.com/aws/amazon-ecs-agent/agent/functional_tests/util" "github.com/aws/amazon-ecs-agent/agent/utils" "github.com/aws/aws-sdk-go/aws" @@ -152,8 +152,8 @@ func TestCommandOverrides(t *testing.T) { agent := RunAgent(t, nil) defer agent.Cleanup() - task, err := agent.StartTaskWithOverrides(t, "simple-exit", []*ecs.ContainerOverride{ - &ecs.ContainerOverride{ + task, err := agent.StartTaskWithOverrides(t, "simple-exit", []*ecsapi.ContainerOverride{ + &ecsapi.ContainerOverride{ Name: strptr("exit"), Command: []*string{strptr("sh"), strptr("-c"), strptr("exit 21")}, }, @@ -396,7 +396,7 @@ func TestAwslogsDriver(t *testing.T) { func TestTelemetry(t *testing.T) { // Try to use a new cluster for this test, ensure no other task metrics for this cluster newClusterName := "ecstest-telemetry-" + uuid.New() - _, err := ECS.CreateCluster(&ecs.CreateClusterInput{ + _, err := ECS.CreateCluster(&ecsapi.CreateClusterInput{ ClusterName: aws.String(newClusterName), }) require.NoError(t, err, "Failed to create cluster") diff --git a/agent/functional_tests/util/utils.go b/agent/functional_tests/util/utils.go index 823ecd75115..0873b1a18a8 100644 --- a/agent/functional_tests/util/utils.go +++ b/agent/functional_tests/util/utils.go @@ -646,3 +646,12 @@ func (agent *TestAgent) SweepTask(task *TestTask) error { return nil } + +// AttributesToMap transforms a list of key, value attributes to return a map +func AttributesToMap(attributes []*ecs.Attribute) map[string]string { + attributeMap := make(map[string]string) + for _, attribute := range attributes { + attributeMap[aws.StringValue(attribute.Name)] = aws.StringValue(attribute.Value) + } + return attributeMap +} diff --git a/agent/functional_tests/util/utils_unix.go b/agent/functional_tests/util/utils_unix.go index a77912bf53d..bcf1805a0e0 100644 --- a/agent/functional_tests/util/utils_unix.go +++ b/agent/functional_tests/util/utils_unix.go @@ -150,6 +150,11 @@ func (agent *TestAgent) StartAgent() error { Cmd: strings.Split(os.Getenv("ECS_FTEST_AGENT_ARGS"), " "), } + // Append ECS_INSTANCE_ATTRIBUTES to dockerConfig + if attr := os.Getenv("ECS_INSTANCE_ATTRIBUTES"); attr != "" { + dockerConfig.Env = append(dockerConfig.Env, "ECS_INSTANCE_ATTRIBUTES="+attr) + } + binds := agent.getBindMounts() hostConfig := &docker.HostConfig{ diff --git a/agent/functional_tests/util/utils_windows.go b/agent/functional_tests/util/utils_windows.go index bd914624a9c..b33e00f7317 100644 --- a/agent/functional_tests/util/utils_windows.go +++ b/agent/functional_tests/util/utils_windows.go @@ -120,10 +120,14 @@ func (agent *TestAgent) StartAgent() error { if TestDirectory := os.Getenv("ECS_WINDOWS_TEST_DIR"); TestDirectory != "" { agentInvoke.Dir = TestDirectory } - agentInvoke.Start() + err := agentInvoke.Start() + if err != nil { + agent.t.Logf("Agent start invocation failed with %s", err.Error()) + return err + } agent.Process = agentInvoke.Process agent.IntrospectionURL = "http://localhost:51678" - err := agent.platformIndependentStartAgent() + err = agent.platformIndependentStartAgent() return err } From 9f4c10a377b8fdc80b7e5165b08f67e523050da6 Mon Sep 17 00:00:00 2001 From: Noah Meyerhans Date: Mon, 20 Feb 2017 10:38:45 -0800 Subject: [PATCH 25/27] Log a bit more info around image pulls. --- agent/engine/docker_task_engine.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/agent/engine/docker_task_engine.go b/agent/engine/docker_task_engine.go index c050a708390..b28a3b061e5 100644 --- a/agent/engine/docker_task_engine.go +++ b/agent/engine/docker_task_engine.go @@ -473,15 +473,18 @@ func (engine *DockerTaskEngine) GetTaskByArn(arn string) (*api.Task, bool) { } func (engine *DockerTaskEngine) pullContainer(task *api.Task, container *api.Container) DockerContainerMetadata { + pullStart := time.Now() + defer seelog.Infof("Finished pulling container %v after %v.", container, time.Since(pullStart).String()) if engine.enableConcurrentPull { + seelog.Infof("Pulling container %v concurrently. Task: %v", container, task) return engine.concurrentPull(task, container) } else { + seelog.Infof("Pulling container %v serially. Task: %v", container, task) return engine.serialPull(task, container) } } func (engine *DockerTaskEngine) concurrentPull(task *api.Task, container *api.Container) DockerContainerMetadata { - log.Info("Pulling container concurrently", "task", task, "container", container) seelog.Debugf("Attempting to obtain ImagePullDeleteLock to pull image - %s", container.Image) ImagePullDeleteLock.RLock() @@ -492,7 +495,6 @@ func (engine *DockerTaskEngine) concurrentPull(task *api.Task, container *api.Co } func (engine *DockerTaskEngine) serialPull(task *api.Task, container *api.Container) DockerContainerMetadata { - log.Info("Pulling container serially", "task", task, "container", container) seelog.Debugf("Attempting to obtain ImagePullDeleteLock to pull image - %s", container.Image) ImagePullDeleteLock.Lock() From 760c64fb7c9d74d25fd39a0fdae30162971b276f Mon Sep 17 00:00:00 2001 From: Noah Meyerhans Date: Wed, 1 Mar 2017 11:35:44 -0800 Subject: [PATCH 26/27] Update log wording to indicate that measured latency includes lock acquisition. --- agent/engine/docker_task_engine.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/engine/docker_task_engine.go b/agent/engine/docker_task_engine.go index b28a3b061e5..20240802149 100644 --- a/agent/engine/docker_task_engine.go +++ b/agent/engine/docker_task_engine.go @@ -474,7 +474,7 @@ func (engine *DockerTaskEngine) GetTaskByArn(arn string) (*api.Task, bool) { func (engine *DockerTaskEngine) pullContainer(task *api.Task, container *api.Container) DockerContainerMetadata { pullStart := time.Now() - defer seelog.Infof("Finished pulling container %v after %v.", container, time.Since(pullStart).String()) + defer seelog.Infof("Finished pulling container %v. Lock acquisition and pull took %v.", container, time.Since(pullStart).String()) if engine.enableConcurrentPull { seelog.Infof("Pulling container %v concurrently. Task: %v", container, task) return engine.concurrentPull(task, container) From 467c3d772d55e02ad618e4901afed1da1173496e Mon Sep 17 00:00:00 2001 From: Adnan Khan Date: Mon, 6 Mar 2017 11:16:12 -0800 Subject: [PATCH 27/27] update agent version to 1.14.1 --- CHANGELOG.md | 9 +++++++++ VERSION | 2 +- agent/version/version.go | 2 +- misc/windows-deploy/user-data.ps1 | 2 +- 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac9d0c5f1ef..25de84119be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## 1.14.1 +* Enhancement - Log completion of image pulls. [#715](https://github.com/aws/amazon-ecs-agent/pull/715) +* Enhancement - Increase start and create timeouts to improve reliability under + some workloads. [#696](https://github.com/aws/amazon-ecs-agent/pull/696) +* Bug - Fixed a bug where throttles on state change reporting could lead to + corrupted state. [#705](https://github.com/aws/amazon-ecs-agent/pull/705) +* Bug - Correct formatting of log messages from tcshandler. [#693](https://github.com/aws/amazon-ecs-agent/pull/693) +* Bug - Fixed an issue where agent could crash. [#692](https://github.com/aws/amazon-ecs-agent/pull/692) + ## 1.14.0 * Feature - Support definition of custom attributes on agent registration. * Feature - Support Docker on Windows Server 2016. diff --git a/VERSION b/VERSION index cd99d386a8d..63e799cf451 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.14.0 \ No newline at end of file +1.14.1 diff --git a/agent/version/version.go b/agent/version/version.go index 9ab35a07dd5..4cd3f24d2a6 100644 --- a/agent/version/version.go +++ b/agent/version/version.go @@ -22,7 +22,7 @@ package version // repository. Only the 'Version' const should change in checked-in source code // Version is the version of the Agent -const Version = "1.14.0" +const Version = "1.14.1" // GitDirty indicates the cleanliness of the git repo when this agent was built const GitDirty = true diff --git a/misc/windows-deploy/user-data.ps1 b/misc/windows-deploy/user-data.ps1 index 6b6ec407ac5..5d1bc4a62b7 100644 --- a/misc/windows-deploy/user-data.ps1 +++ b/misc/windows-deploy/user-data.ps1 @@ -4,7 +4,7 @@ # Set agent env variables for the Machine context (durable) [Environment]::SetEnvironmentVariable("ECS_CLUSTER", "windows", "Machine") [Environment]::SetEnvironmentVariable("ECS_ENABLE_TASK_IAM_ROLE", "false", "Machine") -$agentVersion = 'v1.14.0' +$agentVersion = 'v1.14.1' $agentZipUri = "https://s3.amazonaws.com/amazon-ecs-agent/ecs-agent-windows-$agentVersion.zip" $agentZipMD5Uri = "$agentZipUri.md5"