diff --git a/Gopkg.lock b/Gopkg.lock index 5ffcbfb4cffa1..72093f7918b3b 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -28,28 +28,22 @@ revision = "2cfb68475274527a10701355c739f31dd404718c" [[projects]] - branch = "1.x" name = "github.com/DataDog/zstd" packages = ["."] - revision = "dcab8fb122781db546af7aa75978ae681b6085f5" + revision = "0727e1752ea5db67e0ede4e0fb7168ddc6ead61b" + version = "v1.3.0" [[projects]] name = "github.com/Microsoft/go-winio" packages = ["."] - revision = "78439966b38d69bf38227fbf57ac8a6fee70f69a" - version = "v0.4.5" - -[[projects]] - name = "github.com/Sirupsen/logrus" - packages = ["."] - revision = "55eb11d21d2a31a3cc93838241d04800f52e823d" - version = "v0.7.3" + revision = "7da180ee92d8bd8bb8c37fc560e673e6557c392f" + version = "v0.4.7" [[projects]] branch = "master" name = "github.com/StackExchange/wmi" packages = ["."] - revision = "ea383cf3ba6ec950874b8486cd72356d007c768f" + revision = "5d049714c4a64225c3c79a7cf7d02f7fb5b96338" [[projects]] name = "github.com/aws/aws-sdk-go" @@ -80,8 +74,8 @@ "service/ec2", "service/sts" ] - revision = "e63027ac6e05f6d4ae9f97ce0294d7468ca652da" - version = "v1.10.33" + revision = "1b176c5c6b57adb03bb982c21930e708ebca5a77" + version = "v1.12.70" [[projects]] name = "github.com/beevik/ntp" @@ -91,8 +85,8 @@ [[projects]] name = "github.com/cenkalti/backoff" packages = ["."] - revision = "32cd0c5b3aef12c76ed64aaf678f6c79736be7dc" - version = "v1.0.0" + revision = "61153c768f31ee5f130071d08fc82b85208528de" + version = "v1.1.0" [[projects]] name = "github.com/cihub/seelog" @@ -107,21 +101,23 @@ "pkg/pathutil", "pkg/types" ] - revision = "0520cb9304cb2385f7e72b8bc02d6e4d3257158a" - version = "v3.1.10" + revision = "960f4604bc821241b94906da5fe7f66306a77472" + version = "v3.1.11" [[projects]] name = "github.com/davecgh/go-spew" packages = ["spew"] - revision = "6d212800a42e8ab5c146b8ace3490ee17e5225f9" + revision = "346938d642f2ec3594ed81d874461961cd0faa76" + version = "v1.1.0" [[projects]] name = "github.com/docker/distribution" packages = [ - "digestset", + "digest", "reference" ] - revision = "5cfdfbdce59c0b1854b9d82189e9c6164d54456a" + revision = "48294d928ced5dd9b378f7fd7c6f5da3ff3f2c89" + version = "v2.6.2" [[projects]] name = "github.com/docker/docker" @@ -153,13 +149,27 @@ "sockets", "tlsconfig" ] - revision = "e15c02316c12de00874640cd76311849de2aeed5" + revision = "3ede32e2033de7505e6500d6c868c2b9ed9f169d" + version = "v0.3.0" [[projects]] - branch = "master" name = "github.com/docker/go-units" packages = ["."] revision = "0dadbb0345b35ec7ef35e228dabb8de89a65bf52" + version = "v0.3.2" + +[[projects]] + branch = "master" + name = "github.com/dsnet/compress" + packages = [ + ".", + "bzip2", + "bzip2/internal/sais", + "internal", + "internal/errors", + "internal/prefix" + ] + revision = "cc9eb1d7ad760af14e8f918698f745e80377af4f" [[projects]] name = "github.com/ericchiang/k8s" @@ -195,7 +205,8 @@ "util/intstr", "watch/versioned" ] - revision = "fa30591be9ccaacee478995fbba53c648e901483" + revision = "68b0248f880c5967d1ba7166486cb216af849b15" + version = "v0.4.0" [[projects]] name = "github.com/fatih/color" @@ -206,20 +217,20 @@ [[projects]] name = "github.com/fsnotify/fsnotify" packages = ["."] - revision = "629574ca2a5df945712d3079857300b5e4da0236" - version = "v1.4.2" + revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9" + version = "v1.4.7" [[projects]] branch = "master" name = "github.com/geoffgarside/ber" packages = ["."] - revision = "854377f11dfb81f04121879829bc53487e377739" + revision = "0b763e6b6fb1cb7422c29cd9195a3abf625651fb" [[projects]] name = "github.com/go-ini/ini" packages = ["."] - revision = "20b96f641a5ea98f2f8619ff4f3e061cff4833bd" - version = "v1.28.2" + revision = "32e4c1e6bc4e7d0d8451aa6b75200d19e37a536a" + version = "v1.32.0" [[projects]] name = "github.com/go-ole/go-ole" @@ -227,7 +238,8 @@ ".", "oleutil" ] - revision = "de8695c8edbf8236f30d6e1376e20b198a028d42" + revision = "a41e3c4b706f6ae8dfbff342b06e40fa4d2d0506" + version = "v1.2.1" [[projects]] name = "github.com/gogo/protobuf" @@ -236,13 +248,20 @@ "proto", "protoc-gen-gogo/descriptor" ] - revision = "909568be09de550ed094403c2bf8a261b5bb730a" - version = "v0.3" + revision = "1adfc126b41513cc696b209667c8656ea7aac67c" + version = "v1.0.0" [[projects]] name = "github.com/golang/protobuf" packages = ["proto"] - revision = "4bd1920723d7b7c925de087aa32e2187708897f7" + revision = "925541529c1fa6821df4e44ce2723319eb2be768" + version = "v1.0.0" + +[[projects]] + branch = "master" + name = "github.com/golang/snappy" + packages = ["."] + revision = "553a641470496b2327abcac10b36396bd98e45c9" [[projects]] branch = "master" @@ -259,19 +278,20 @@ [[projects]] name = "github.com/gorilla/mux" packages = ["."] - revision = "bcd8bc72b08df0f70df986b97f95590779502d31" - version = "v1.4.0" + revision = "53c1911da2b537f792e7cafcb446b05ffe33b996" + version = "v1.6.1" [[projects]] name = "github.com/hashicorp/consul" packages = ["api"] - revision = "5473255f9816e2b64d52cc054470d600b007d821" + revision = "48f3dd5642374d079f5a64359023fb8318eb81cc" + version = "v1.0.3" [[projects]] branch = "master" name = "github.com/hashicorp/go-cleanhttp" packages = ["."] - revision = "3573b8b52aa7b37b9358d966a898feb387f62437" + revision = "d5fe4b57a186c716b0e00b8c301cbd9b4182694d" [[projects]] branch = "master" @@ -280,6 +300,7 @@ revision = "6bb64b370b90e7ef1fa532be9e591a81c3493e00" [[projects]] + branch = "master" name = "github.com/hashicorp/hcl" packages = [ ".", @@ -292,12 +313,13 @@ "json/scanner", "json/token" ] - revision = "7fa7fff964d035e8a162cce3a164b3ad02ad651b" + revision = "23c074d0eceb2b8a5bfdbb271ab780cde70f05a8" [[projects]] name = "github.com/hashicorp/serf" packages = ["coordinate"] - revision = "91fd53b1d3e624389ed9a295a3fa380e5c7b9dfc" + revision = "d6574a5bb1226678d7010325fb6c985db20ee458" + version = "v0.8.1" [[projects]] branch = "master" @@ -314,16 +336,10 @@ revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75" version = "v1.0" -[[projects]] - name = "github.com/jhoonb/archivex" - packages = ["."] - revision = "be4efa7ec0c38ab76d56037014c90d48d6b13037" - [[projects]] name = "github.com/jmespath/go-jmespath" packages = ["."] - revision = "3433f3ea46d9f8019119e7dd41274e112a2359a9" - version = "0.2.2" + revision = "0b12d6b5" [[projects]] name = "github.com/k-sone/snmpgo" @@ -332,9 +348,10 @@ version = "v3.2.0" [[projects]] + branch = "master" name = "github.com/kardianos/osext" packages = ["."] - revision = "9d302b58e975387d0b4d9be876622c86cefe64be" + revision = "ae77be60afb1dcacde03767a8c37337fad28ac14" [[projects]] branch = "master" @@ -345,7 +362,8 @@ [[projects]] name = "github.com/magiconair/properties" packages = ["."] - revision = "51463bfca2576e06c62a8504b5c0f06d61312647" + revision = "d419a98cdbed11a922bf76f257b7c4be79b50e73" + version = "v1.7.4" [[projects]] name = "github.com/mattn/go-colorable" @@ -359,6 +377,11 @@ revision = "0360b2af4f38e8d38c7fce2a9f4e702702d73a39" version = "v0.0.3" +[[projects]] + name = "github.com/mholt/archiver" + packages = ["."] + revision = "26cf5bb32d07aa4e8d0de15f56ce516f4641d7df" + [[projects]] branch = "master" name = "github.com/mitchellh/go-homedir" @@ -366,52 +389,64 @@ revision = "b8bc1bf767474819792c23f32d8286a45736f1c6" [[projects]] + branch = "master" name = "github.com/mitchellh/mapstructure" packages = ["."] - revision = "cc8532a8e9a55ea36402aa21efdf403a60d34096" + revision = "b4575eea38cca1123ec2dc90c26529b5c5acfcff" [[projects]] + branch = "master" name = "github.com/mitchellh/reflectwalk" packages = ["."] - revision = "417edcfd99a4d472c262e58f22b4bfe97580f03e" + revision = "63d60e9d0dbc60cf9164e6510889b0db6683d98c" [[projects]] - name = "github.com/opencontainers/go-digest" + branch = "master" + name = "github.com/nwaples/rardecode" packages = ["."] - revision = "aa2ec055abd10d26d539eb630a92241b781ce4bc" - version = "v1.0.0-rc0" + revision = "e06696f847aeda6f39a8f0b7cdff193b7690aef6" [[projects]] name = "github.com/patrickmn/go-cache" packages = ["."] - revision = "1881a9bccb818787f68c52bfba648c6cf34c34fa" - version = "v2.0.0" + revision = "a3647f8e31d79543b2d0f0ae2fe5c379d72cedc0" + version = "v2.1.0" [[projects]] - name = "github.com/pelletier/go-buffruneio" + name = "github.com/pelletier/go-toml" packages = ["."] - revision = "c37440a7cf42ac63b919c752ca73a85067e05992" - version = "v0.2.0" + revision = "acdc4509485b587f5e675510c4f2c63e90ff68a8" + version = "v1.1.0" [[projects]] - name = "github.com/pelletier/go-toml" + name = "github.com/pierrec/lz4" packages = ["."] - revision = "fe206efb84b2bc8e8cfafe6b4c1826622be969e3" + revision = "2fcda4cb7018ce05a25959d2fe08c83e3329f169" + version = "v1.1" + +[[projects]] + name = "github.com/pierrec/xxHash" + packages = ["xxHash32"] + revision = "f051bb7f1d1aaf1b5a665d74fb6b0217712c69f7" + version = "v0.1.1" [[projects]] name = "github.com/pkg/errors" packages = ["."] - revision = "ff09b135c25aae272398c51a07235b90a75aa4f0" + revision = "645ef00459ed84a119197bfb8d8205042c6df63d" + version = "v0.8.0" [[projects]] name = "github.com/pmezard/go-difflib" packages = ["difflib"] - revision = "d8ed2627bdf02c080bf22230dbb337003b7aba2d" + revision = "792786c7400a136282c1664665ae0a8db921c6c2" + version = "v1.0.0" [[projects]] + branch = "master" name = "github.com/samuel/go-zookeeper" packages = ["zk"] - revision = "1d7be4effb13d2d908342d349d71a284a7542693" + revision = "c4fab1ac1bec58281ad0667dc3f0907a9476ac47" [[projects]] branch = "master" @@ -431,8 +466,8 @@ "net", "process" ] - revision = "a452de7c734a0fa0f16d2e5725b0fa5934d9fbec" - version = "v2.17.08" + revision = "c432be29ccce470088d07eea25b3ea7e68a8afbb" + version = "v2.18.01" [[projects]] branch = "master" @@ -441,13 +476,13 @@ revision = "bb4de0191aa41b5507caa14b0650cdbddcd9280b" [[projects]] - branch = "master" name = "github.com/spf13/afero" packages = [ ".", "mem" ] - revision = "9be650865eab0c12963d8753212f4f9c66cdcf12" + revision = "bb8f1927f2a9d3ab41c9340aa034f6b803f4359c" + version = "v1.0.2" [[projects]] name = "github.com/spf13/cast" @@ -458,27 +493,32 @@ [[projects]] name = "github.com/spf13/cobra" packages = ["."] - revision = "7b1b6e8dc027253d45fc029bc269d1c019f83a34" + revision = "7b2c5ac9fc04fc5efafb60700713d4fa609b777b" + version = "v0.0.1" [[projects]] + branch = "master" name = "github.com/spf13/jwalterweatherman" packages = ["."] - revision = "fa7ca7e836cf3a8bb4ebf799f472c12d7e903d66" + revision = "7c0cea34c8ece3fbeb2b27ab9b59511d360fb394" [[projects]] name = "github.com/spf13/pflag" packages = ["."] - revision = "f1d95a35e132e8a1868023a08932b14f0b8b8fcb" + revision = "e57e3eeb33f795204c1ca35f56c44f83227c6e66" + version = "v1.0.0" [[projects]] name = "github.com/spf13/viper" packages = ["."] - revision = "0967fc9aceab2ce9da34061253ac10fb99bba5b2" + revision = "25b30aa063fc18e48662b86996252eabdcf2f0c7" + version = "v1.0.0" [[projects]] name = "github.com/stretchr/objx" packages = ["."] - revision = "cbeaeb16a013161a98496fad62933b1d21786672" + revision = "facf9a85c22f48d2f52f2380e4efce1768749a89" + version = "v0.1" [[projects]] name = "github.com/stretchr/testify" @@ -488,21 +528,33 @@ "require", "suite" ] - revision = "69483b4bd14f5845b5a1e55bca19e954e827f1d0" - version = "v1.1.4" + revision = "12b6f73e6084dad08a7c6e575284b177ecafbc71" + version = "v1.2.1" [[projects]] name = "github.com/ugorji/go" packages = ["codec"] - revision = "ded73eae5db7e7a0ef6f55aace87a2873c5d2b74" + revision = "8c0409fcbb70099c748d71f714529204975f6c3f" + +[[projects]] + name = "github.com/ulikunitz/xz" + packages = [ + ".", + "internal/hash", + "internal/xlog", + "lzma" + ] + revision = "0c6b41e72360850ca4f98dc341fd999726ea007f" + version = "v0.5.4" [[projects]] name = "github.com/urfave/negroni" packages = ["."] - revision = "fde5e16d32adc7ad637e9cd9ad21d4ebc6192535" - version = "v0.2.0" + revision = "5dbbc83f748fc3ad38585842b0aedab546d0ea1e" + version = "v0.3.0" [[projects]] + branch = "master" name = "golang.org/x/net" packages = [ "context", @@ -513,9 +565,10 @@ "lex/httplex", "proxy" ] - revision = "f2499483f923065a842d38eb4c7f1927e6fc6e6d" + revision = "2fb46b16b8dda405028c50f7c7f0f9dd1fa6bfb1" [[projects]] + branch = "master" name = "golang.org/x/sys" packages = [ "unix", @@ -526,19 +579,28 @@ "windows/svc/eventlog", "windows/svc/mgr" ] - revision = "8f0908ab3b2457e2e15403d3697c9ef5cb4b57a9" + revision = "37707fdb30a5b38865cfb95e5aab41707daec7fd" [[projects]] + branch = "master" name = "golang.org/x/text" packages = [ + "collate", + "collate/build", + "internal/colltab", "internal/gen", + "internal/tag", "internal/triegen", "internal/ucd", + "language", + "secure/bidirule", "transform", + "unicode/bidi", "unicode/cldr", - "unicode/norm" + "unicode/norm", + "unicode/rangetable" ] - revision = "2910a502d2bf9e43193af9d68ca516529614eed3" + revision = "e19ae1496984b1c655b8044a65c0300a3c878dd3" [[projects]] branch = "master" @@ -548,22 +610,23 @@ "go/buildutil", "go/loader" ] - revision = "9badcbe49be523255546b669968041d707ece12e" + revision = "66487607e2081c7c2af2281c62c14ee000d5024b" [[projects]] + branch = "v2" name = "gopkg.in/yaml.v2" packages = ["."] - revision = "cd8b52f8269e0feb286dfeef29f8fe4d5b397e0b" + revision = "d670f9405373e636a5a2765eea47fac0c9bc91a4" [[projects]] name = "gopkg.in/zorkian/go-datadog-api.v2" packages = ["."] - revision = "659bba33a1626ab76f393d6878dc3e4f13253f6d" - version = "v2.3" + revision = "8d504de7b41981e0075444686fa389708bce2dbc" + version = "v2.8" [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "aa71cac4e8edd4c7bef396db630a9af0859f76b623fdeaba65da136be4bd0183" + inputs-digest = "e35db1c219179b3e276941f11a2ec419965647e4668b56aed2eccf6a67aae55a" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 1f9b572cbfddc..3edecd0a8b364 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -45,6 +45,10 @@ name = "github.com/gorilla/mux" version = "1.1.0" +[[constraint]] + name = "github.com/mholt/archiver" + revision = "26cf5bb32d07aa4e8d0de15f56ce516f4641d7df" + [[constraint]] name = "github.com/jhoonb/archivex" revision = "be4efa7ec0c38ab76d56037014c90d48d6b13037" @@ -83,6 +87,10 @@ name = "github.com/stretchr/testify" version = "1.1.3" +[[override]] + name = "github.com/ugorji/go" + revision = "8c0409fcbb70099c748d71f714529204975f6c3f" + [[constraint]] name = "gopkg.in/yaml.v2" diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index dfed0493a9f10..af6068baa0780 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -31,6 +31,7 @@ core,github.com/jquery/jquery,MIT core,github.com/k-sone/snmpgo,MIT core,github.com/kardianos/osext,BSD-3-Clause core,github.com/magiconair/properties,BSD-2-Clause +core,github.com/mholt/archiver,MIT core,github.com/Microsoft/go-winio,MIT core,github.com/mitchellh/mapstructure,MIT core,github.com/mitchellh/reflectwalk,MIT diff --git a/pkg/flare/archive.go b/pkg/flare/archive.go index c78a8c79e91ee..58f6a7806cff3 100644 --- a/pkg/flare/archive.go +++ b/pkg/flare/archive.go @@ -8,8 +8,11 @@ package flare import ( "bufio" "bytes" + "crypto/rand" + "encoding/hex" "encoding/json" "expvar" + "io/ioutil" "os" "path/filepath" "sort" @@ -22,7 +25,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/status/health" "github.com/DataDog/datadog-agent/pkg/util" - "github.com/jhoonb/archivex" + "github.com/mholt/archiver" yaml "gopkg.in/yaml.v2" ) @@ -31,7 +34,7 @@ type SearchPaths map[string]string // CreateArchive packages up the files func CreateArchive(local bool, distPath, pyChecksPath, logFilePath string) (string, error) { - zipFilePath := mkFilePath() + zipFilePath := getArchivePath() confSearchPaths := SearchPaths{ "": config.Datadog.GetString("confd_path"), "dist": filepath.Join(distPath, "conf.d"), @@ -41,8 +44,19 @@ func CreateArchive(local bool, distPath, pyChecksPath, logFilePath string) (stri } func createArchive(zipFilePath string, local bool, confSearchPaths SearchPaths, logFilePath string) (string, error) { - zipFile := new(archivex.ZipFile) - zipFile.Create(zipFilePath) + b := make([]byte, 10) + _, err := rand.Read(b) + if err != nil { + return "", err + } + + dirName := hex.EncodeToString([]byte(b)) + tempDir, err := ioutil.TempDir("", dirName) + if err != nil { + return "", err + } + + defer os.RemoveAll(tempDir) // Get hostname, if there's an error in getting the hostname, // set the hostname to unknown @@ -51,82 +65,108 @@ func createArchive(zipFilePath string, local bool, confSearchPaths SearchPaths, hostname = "unknown" } - defer zipFile.Close() - if local { - zipFile.Add(filepath.Join(hostname, "local"), []byte{}) + f := filepath.Join(tempDir, hostname, "local") + + err = ensureParentDirsExist(f) + if err != nil { + return "", err + } + + err = ioutil.WriteFile(f, []byte{}, os.ModePerm) + if err != nil { + return "", err + } } else { // The Status will be unavailable unless the agent is running. // Only zip it up if the agent is running - err = zipStatusFile(zipFile, hostname) + err = zipStatusFile(tempDir, hostname) if err != nil { return "", err } } - err = zipLogFiles(zipFile, hostname, logFilePath) + err = zipLogFiles(tempDir, hostname, logFilePath) if err != nil { return "", err } - err = zipConfigFiles(zipFile, hostname, confSearchPaths) + err = zipConfigFiles(tempDir, hostname, confSearchPaths) if err != nil { return "", err } - err = zipExpVar(zipFile, hostname) + err = zipExpVar(tempDir, hostname) if err != nil { return "", err } - err = zipDiagnose(zipFile, hostname) + err = zipDiagnose(tempDir, hostname) if err != nil { return "", err } - err = zipEnvvars(zipFile, hostname) + err = zipEnvvars(tempDir, hostname) if err != nil { return "", err } - err = zipConfigCheck(zipFile, hostname) + err = zipConfigCheck(tempDir, hostname) if err != nil { return "", err } - err = zipHealth(zipFile, hostname) + err = zipHealth(tempDir, hostname) if err != nil { return "", err } if config.IsContainerized() { - err = zipDockerSelfInspect(zipFile, hostname) + err = zipDockerSelfInspect(tempDir, hostname) if err != nil { return "", err } } + + err = archiver.Zip.Make(zipFilePath, []string{filepath.Join(tempDir, hostname)}) + if err != nil { + return "", err + } + return zipFilePath, nil } -func zipStatusFile(zipFile *archivex.ZipFile, hostname string) error { +func zipStatusFile(tempDir, hostname string) error { // Grab the status s, err := status.GetAndFormatStatus() if err != nil { return err } + // Clean it up cleaned, err := credentialsCleanerBytes(s) if err != nil { return err } - // Add it to the zipfile - zipFile.Add(filepath.Join(hostname, "status.log"), cleaned) + + f := filepath.Join(tempDir, hostname, "status.log") + + err = ensureParentDirsExist(f) + if err != nil { + return err + } + + err = ioutil.WriteFile(f, cleaned, os.ModePerm) + if err != nil { + return err + } + return err } -func zipLogFiles(zipFile *archivex.ZipFile, hostname, logFilePath string) error { +func zipLogFiles(tempDir, hostname, logFilePath string) error { logFileDir := filepath.Dir(logFilePath) - err := filepath.Walk(logFileDir, func(path string, f os.FileInfo, err error) error { + err := filepath.Walk(logFileDir, func(src string, f os.FileInfo, err error) error { if f == nil { return nil } @@ -135,8 +175,8 @@ func zipLogFiles(zipFile *archivex.ZipFile, hostname, logFilePath string) error } if filepath.Ext(f.Name()) == ".log" || getFirstSuffix(f.Name()) == ".log" { - fileName := filepath.Join(hostname, "logs", f.Name()) - return zipFile.AddFileWithName(fileName, path) + dst := filepath.Join(tempDir, hostname, "logs", f.Name()) + return util.CopyFileAll(src, dst) } return nil }) @@ -144,7 +184,7 @@ func zipLogFiles(zipFile *archivex.ZipFile, hostname, logFilePath string) error return err } -func zipExpVar(zipFile *archivex.ZipFile, hostname string) error { +func zipExpVar(tempDir, hostname string) error { var variables = make(map[string]interface{}) expvar.Do(func(kv expvar.KeyValue) { var variable = make(map[string]interface{}) @@ -160,11 +200,20 @@ func zipExpVar(zipFile *archivex.ZipFile, hostname string) error { if err != nil { return err } + cleanedYAML, err := credentialsCleanerBytes(yamlValue) if err != nil { return err } - err = zipFile.Add(filepath.Join(hostname, "expvar", key), cleanedYAML) + + f := filepath.Join(tempDir, hostname, "expvar", key) + + err = ensureParentDirsExist(f) + if err != nil { + return err + } + + err = ioutil.WriteFile(f, cleanedYAML, os.ModePerm) if err != nil { return err } @@ -173,7 +222,7 @@ func zipExpVar(zipFile *archivex.ZipFile, hostname string) error { return nil } -func zipConfigFiles(zipFile *archivex.ZipFile, hostname string, confSearchPaths SearchPaths) error { +func zipConfigFiles(tempDir, hostname string, confSearchPaths SearchPaths) error { c, err := yaml.Marshal(config.Datadog.AllSettings()) if err != nil { return err @@ -183,12 +232,20 @@ func zipConfigFiles(zipFile *archivex.ZipFile, hostname string, confSearchPaths if err != nil { return err } - err = zipFile.Add(filepath.Join(hostname, "runtime_config_dump.yaml"), cleaned) + + f := filepath.Join(tempDir, hostname, "runtime_config_dump.yaml") + + err = ensureParentDirsExist(f) if err != nil { return err } - err = walkConfigFilePaths(zipFile, hostname, confSearchPaths) + err = ioutil.WriteFile(f, cleaned, os.ModePerm) + if err != nil { + return err + } + + err = walkConfigFilePaths(tempDir, hostname, confSearchPaths) if err != nil { return err } @@ -196,17 +253,25 @@ func zipConfigFiles(zipFile *archivex.ZipFile, hostname string, confSearchPaths if config.Datadog.ConfigFileUsed() != "" { // zip up the config file that was actually used, if one exists filePath := config.Datadog.ConfigFileUsed() + // Check if the file exists - _, e := os.Stat(filePath) - if e == nil { - file, e := credentialsCleanerFile(filePath) + _, err := os.Stat(filePath) + if err == nil { + cleaned, err = credentialsCleanerFile(filePath) if err != nil { - return e + return err } - fileName := filepath.Join(hostname, "etc", "datadog.yaml") - e = zipFile.Add(fileName, file) - if e != nil { - return e + + f = filepath.Join(tempDir, hostname, "etc", "datadog.yaml") + + err = ensureParentDirsExist(f) + if err != nil { + return err + } + + err = ioutil.WriteFile(f, cleaned, os.ModePerm) + if err != nil { + return err } } } @@ -214,14 +279,21 @@ func zipConfigFiles(zipFile *archivex.ZipFile, hostname string, confSearchPaths return err } -func zipDiagnose(zipFile *archivex.ZipFile, hostname string) error { +func zipDiagnose(tempDir, hostname string) error { var b bytes.Buffer writer := bufio.NewWriter(&b) diagnose.RunAll(writer) writer.Flush() - err := zipFile.Add(filepath.Join(hostname, "diagnose.log"), b.Bytes()) + f := filepath.Join(tempDir, hostname, "diagnose.log") + + err := ensureParentDirsExist(f) + if err != nil { + return err + } + + err = ioutil.WriteFile(f, b.Bytes(), os.ModePerm) if err != nil { return err } @@ -229,14 +301,21 @@ func zipDiagnose(zipFile *archivex.ZipFile, hostname string) error { return nil } -func zipConfigCheck(zipFile *archivex.ZipFile, hostname string) error { +func zipConfigCheck(tempDir, hostname string) error { var b bytes.Buffer writer := bufio.NewWriter(&b) GetConfigCheck(writer, true) writer.Flush() - err := zipFile.Add(filepath.Join(hostname, "config-check.log"), b.Bytes()) + f := filepath.Join(tempDir, hostname, "config-check.log") + + err := ensureParentDirsExist(f) + if err != nil { + return err + } + + err = ioutil.WriteFile(f, b.Bytes(), os.ModePerm) if err != nil { return err } @@ -244,7 +323,7 @@ func zipConfigCheck(zipFile *archivex.ZipFile, hostname string) error { return nil } -func zipHealth(zipFile *archivex.ZipFile, hostname string) error { +func zipHealth(tempDir, hostname string) error { s := health.GetStatus() sort.Strings(s.Healthy) sort.Strings(s.Unhealthy) @@ -253,7 +332,15 @@ func zipHealth(zipFile *archivex.ZipFile, hostname string) error { if err != nil { return err } - err = zipFile.Add(filepath.Join(hostname, "health.yaml"), yamlValue) + + f := filepath.Join(tempDir, hostname, "health.yaml") + + err = ensureParentDirsExist(f) + if err != nil { + return err + } + + err = ioutil.WriteFile(f, yamlValue, os.ModePerm) if err != nil { return err } @@ -261,9 +348,9 @@ func zipHealth(zipFile *archivex.ZipFile, hostname string) error { return nil } -func walkConfigFilePaths(zipFile *archivex.ZipFile, hostname string, confSearchPaths SearchPaths) error { +func walkConfigFilePaths(tempDir, hostname string, confSearchPaths SearchPaths) error { for prefix, filePath := range confSearchPaths { - err := filepath.Walk(filePath, func(path string, f os.FileInfo, err error) error { + err := filepath.Walk(filePath, func(src string, f os.FileInfo, err error) error { if f == nil { return nil } @@ -276,13 +363,23 @@ func walkConfigFilePaths(zipFile *archivex.ZipFile, hostname string, confSearchP } if getFirstSuffix(f.Name()) == ".yaml" || filepath.Ext(f.Name()) == ".yaml" { - baseName := strings.Replace(path, filePath, "", 1) - fileName := filepath.Join(hostname, "etc", "confd", prefix, baseName) - file, err := credentialsCleanerFile(path) + cleaned, err := credentialsCleanerFile(src) + if err != nil { + return err + } + + baseName := strings.Replace(src, filePath, "", 1) + f := filepath.Join(tempDir, hostname, "etc", "confd", prefix, baseName) + + err = ensureParentDirsExist(f) + if err != nil { + return err + } + + err = ioutil.WriteFile(f, cleaned, os.ModePerm) if err != nil { return err } - return zipFile.Add(fileName, file) } return nil @@ -297,11 +394,20 @@ func walkConfigFilePaths(zipFile *archivex.ZipFile, hostname string, confSearchP return nil } +func ensureParentDirsExist(p string) error { + err := os.MkdirAll(filepath.Dir(p), os.ModePerm) + if err != nil { + return err + } + + return nil +} + func getFirstSuffix(s string) string { return filepath.Ext(strings.TrimSuffix(s, filepath.Ext(s))) } -func mkFilePath() string { +func getArchivePath() string { dir := os.TempDir() t := time.Now() timeString := t.Format("2006-01-02-15-04-05") diff --git a/pkg/flare/archive_docker.go b/pkg/flare/archive_docker.go index 3c75c5b021769..0afc7ea2db724 100644 --- a/pkg/flare/archive_docker.go +++ b/pkg/flare/archive_docker.go @@ -10,14 +10,15 @@ package flare import ( "bytes" "encoding/json" + "io/ioutil" + "os" "path/filepath" "regexp" "github.com/DataDog/datadog-agent/pkg/util/docker" - "github.com/jhoonb/archivex" ) -func zipDockerSelfInspect(zipFile *archivex.ZipFile, hostname string) error { +func zipDockerSelfInspect(tempDir, hostname string) error { du, err := docker.GetDockerUtil() if err != nil { return err @@ -50,9 +51,17 @@ func zipDockerSelfInspect(zipFile *archivex.ZipFile, hostname string) error { return []byte(shaResolvedInspect) }) - err = zipFile.Add(filepath.Join(hostname, "docker_inspect.log"), cleaned) + f := filepath.Join(tempDir, hostname, "docker_inspect.log") + + err = os.MkdirAll(filepath.Dir(f), os.ModePerm) + if err != nil { + return err + } + + err = ioutil.WriteFile(f, cleaned, os.ModePerm) if err != nil { return err } + return err } diff --git a/pkg/flare/archive_nodocker.go b/pkg/flare/archive_nodocker.go index 31a8633137ce5..9a63b7d0f6c50 100644 --- a/pkg/flare/archive_nodocker.go +++ b/pkg/flare/archive_nodocker.go @@ -7,10 +7,6 @@ package flare -import ( - "github.com/jhoonb/archivex" -) - -func zipDockerSelfInspect(zipFile *archivex.ZipFile, hostname string) error { +func zipDockerSelfInspect(tempDir, hostname string) error { return nil } diff --git a/pkg/flare/archive_test.go b/pkg/flare/archive_test.go index 1d019287445b4..6129710335e1b 100644 --- a/pkg/flare/archive_test.go +++ b/pkg/flare/archive_test.go @@ -18,7 +18,7 @@ func TestCreateArchive(t *testing.T) { common.SetupConfig("./test") config.Datadog.Set("confd_path", "./test/confd") config.Datadog.Set("log_file", "./test/logs/agent.log") - zipFilePath := mkFilePath() + zipFilePath := getArchivePath() filePath, err := createArchive(zipFilePath, true, SearchPaths{}, "") assert.Nil(t, err) @@ -37,7 +37,7 @@ func TestCreateArchiveBadConfig(t *testing.T) { **/ common.SetupConfig("") - zipFilePath := mkFilePath() + zipFilePath := getArchivePath() filePath, err := createArchive(zipFilePath, true, SearchPaths{}, "") assert.Nil(t, err) diff --git a/pkg/flare/envvars.go b/pkg/flare/envvars.go index ec6ec40fd873d..4d11110e81657 100644 --- a/pkg/flare/envvars.go +++ b/pkg/flare/envvars.go @@ -7,11 +7,10 @@ package flare import ( "bytes" + "io/ioutil" "os" "path/filepath" "strings" - - "github.com/jhoonb/archivex" ) var envvarPrefixWhitelist = []string{ @@ -47,7 +46,7 @@ func getWhitelistedEnvvars() []string { // zipEnvvars collects whitelisted envvars that can affect the agent's // behaviour while not being handled by viper -func zipEnvvars(zipFile *archivex.ZipFile, hostname string) error { +func zipEnvvars(tempDir, hostname string) error { envvars := getWhitelistedEnvvars() if len(envvars) == 0 { // Don't create the file if we have nothing @@ -66,9 +65,17 @@ func zipEnvvars(zipFile *archivex.ZipFile, hostname string) error { return err } - err = zipFile.Add(filepath.Join(hostname, "envvars.log"), cleaned) + f := filepath.Join(tempDir, hostname, "envvars.log") + + err = os.MkdirAll(filepath.Dir(f), os.ModePerm) if err != nil { return err } + + err = ioutil.WriteFile(f, cleaned, os.ModePerm) + if err != nil { + return err + } + return err } diff --git a/releasenotes/notes/fix-flare-archive-structure-7c892ad559f75e59.yaml b/releasenotes/notes/fix-flare-archive-structure-7c892ad559f75e59.yaml new file mode 100644 index 0000000000000..75d440e3ade0e --- /dev/null +++ b/releasenotes/notes/fix-flare-archive-structure-7c892ad559f75e59.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + Fixed the structure of the Flare archive on all platforms.