From acab5e6c7a3b229d166766a2be19a618dc56cd84 Mon Sep 17 00:00:00 2001 From: Yusuke KUOKA Date: Tue, 4 Jun 2019 19:21:09 +0900 Subject: [PATCH] feat: remote state files This change enhances helmfile to accept terraform-module-like URLs in nested state files a.k.a sub-helmfiles. ```yaml helmfiles: - # Terraform-module-like URL for importing a remote directory and use a file in it as a nested-state file # The nested-state file is locally checked-out along with the remote directory containing it. # Therefore all the local paths in the file are resolved relative to the file path: git::https://github.com/cloudposse/helmfiles.git@releases/kiam.yaml?ref=0.40.0 ``` The URL isn't equivalent to terraform module sources. The difference is that we use `@` to distinguish between (1) the path to the repository and directory containing the state file and (2) the path to the state file being loaded. This distinction provides us enough fleibiity to instruct helmfile to check-out necessary and sufficient directory to make the state file works. Under the hood, it uses [hashicorp/go-getter](https://github.com/hashicorp/go-getter), that is used for [terraform module sources](https://www.terraform.io/docs/modules/sources.html) as well. Only the git provider without authentication like git-credentials helper is tested. But theoretically go-getter providers should work. Please feel free to test the provider of your choice and contribute documentation or instruction to use it :) Resolves #347 --- README.md | 4 + go.mod | 3 +- go.sum | 167 +++++++++++++++++++ pkg/app/app.go | 32 ++++ pkg/app/app_test.go | 33 ++-- pkg/app/two_pass_renderer_test.go | 5 +- pkg/remote/remote.go | 242 ++++++++++++++++++++++++++++ pkg/remote/remote_test.go | 94 +++++++++++ pkg/state/create_test.go | 3 +- pkg/state/state.go | 6 + pkg/state/state_test.go | 9 +- pkg/{state => testhelper}/testfs.go | 9 +- 12 files changed, 579 insertions(+), 28 deletions(-) create mode 100644 pkg/remote/remote.go create mode 100644 pkg/remote/remote_test.go rename pkg/{state => testhelper}/testfs.go (93%) diff --git a/README.md b/README.md index 4cbc13838..3476b0fcb 100644 --- a/README.md +++ b/README.md @@ -184,6 +184,10 @@ helmfiles: - # All the nested state files under `helmfiles:` is processed in the order of definition. # So it can be used for preparation for your main `releases`. An example would be creating CRDs required by `reelases` in the parent state file. path: path/to/mycrd.helmfile.yaml +- # Terraform-module-like URL for importing a remote directory and use a file in it as a nested-state file + # The nested-state file is locally checked-out along with the remote directory containing it. + # Therefore all the local paths in the file are resolved relative to the file + path: git::https://github.com/cloudposse/helmfiles.git@releases/kiam.yaml?ref=0.40.0 # # Advanced Configuration: Environments diff --git a/go.mod b/go.mod index a887beabe..044e27426 100644 --- a/go.mod +++ b/go.mod @@ -8,16 +8,15 @@ require ( github.com/aokoli/goutils v1.0.1 // indirect github.com/google/go-cmp v0.3.0 github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c // indirect + github.com/hashicorp/go-getter v1.3.0 github.com/huandu/xstrings v1.0.0 // indirect github.com/imdario/mergo v0.3.6 - github.com/mattn/go-runewidth v0.0.4 // indirect github.com/pkg/errors v0.8.1 // indirect github.com/tatsushid/go-prettytable v0.0.0-20141013043238-ed2d14c29939 github.com/urfave/cli v0.0.0-20160620154522-6011f165dc28 go.uber.org/atomic v1.3.2 // indirect go.uber.org/multierr v1.1.0 // indirect go.uber.org/zap v1.8.0 - golang.org/x/crypto v0.0.0-20180403160946-b2aa35443fbc // indirect gopkg.in/yaml.v2 v2.2.1 gotest.tools v2.2.0+incompatible ) diff --git a/go.sum b/go.sum index 605759462..577638823 100644 --- a/go.sum +++ b/go.sum @@ -1,37 +1,204 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.36.0 h1:+aCSj7tOo2LODWVEuZDZeGCckdt6MlSF+X/rB3wUiS8= +cloud.google.com/go v0.36.0/go.mod h1:RUoy9p/M4ge0HzT8L+SDZ8jg+Q6fth0CiBuhFJpSV40= +dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= +dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU= +dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= +dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= +git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Masterminds/semver v1.4.1 h1:CaDA1wAoM3rj9sAFyyZP37LloExUzxFGYt+DqJ870JA= github.com/Masterminds/semver v1.4.1/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/sprig v2.15.0+incompatible h1:0gSxPGWS9PAr7U2NsQ2YQg6juRDINkUyuvbb4b2Xm8w= github.com/Masterminds/sprig v2.15.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= github.com/Masterminds/sprig v2.16.0+incompatible h1:QZbMUPxRQ50EKAq3LFMnxddMu88/EUUG3qmxwtDmPsY= github.com/Masterminds/sprig v2.16.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/aokoli/goutils v1.0.1 h1:7fpzNGoJ3VA8qcrm++XEE1QUe0mIwNeLa02Nwq7RDkg= github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g9DP+DQ= +github.com/aws/aws-sdk-go v1.15.78 h1:LaXy6lWR0YK7LKyuU0QWy2ws/LWTPfYV/UgfiBu4tvY= +github.com/aws/aws-sdk-go v1.15.78/go.mod h1:E3/ieXAlvM0XWO57iftYVDLLvQ824smPP3ATZkfNZeM= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas= +github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4= +github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= +github.com/cheggaaa/pb v1.0.27/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c h1:jWtZjFEUE/Bz0IeIhqCnyZ3HG6KRXSntXe4SjtuTH7c= github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go v2.0.0+incompatible h1:j0GKcs05QVmm7yesiZq2+9cxHkNK9YM6zKx4D2qucQU= +github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= +github.com/googleapis/gax-go/v2 v2.0.3 h1:siORttZ36U2R/WjiJuDz8znElWBiAlO9rVt+mqJt0Cc= +github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= +github.com/hashicorp/go-cleanhttp v0.5.0 h1:wvCrVc9TjDls6+YGAF2hAifE1E5U1+b4tH6KdvN3Gig= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-getter v1.3.0 h1:pFMSFlI9l5NaeuzkpE3L7BYk9qQ9juTAgXW/H0cqxcU= +github.com/hashicorp/go-getter v1.3.0/go.mod h1:/O1k/AizTN0QmfEKknCYGvICeyKUDqCYA8vvWtGWDeQ= +github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo= +github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoDHxNAB65b+Rj1I= +github.com/hashicorp/go-version v1.1.0 h1:bPIoEKD27tNdebFGGxxYwcL4nepeY4j1QP23PFRGzg0= +github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/huandu/xstrings v1.0.0 h1:pO2K/gKgKaat5LdpAhxhluX2GPQMaI3W5FUz/I/UnWk= github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo= github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= +github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 h1:12VvqtR6Aowv3l/EQUlocDHW2Cp4G9WJVH7uyH8QFJE= +github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= +github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= +github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= +github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= +github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= +github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0= +github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= +github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= +github.com/shurcooL/gofontwoff v0.0.0-20180329035133-29b52fc0a18d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw= +github.com/shurcooL/gopherjslib v0.0.0-20160914041154-feb6d3990c2c/go.mod h1:8d3azKNyqcHP1GaQE/c6dDgjkgSx2BZ4IoEi4F1reUI= +github.com/shurcooL/highlight_diff v0.0.0-20170515013008-09bb4053de1b/go.mod h1:ZpfEhSmds4ytuByIcDnOLkTHGUI6KNqRNPDLHDk+mUU= +github.com/shurcooL/highlight_go v0.0.0-20181028180052-98c3abbbae20/go.mod h1:UDKB5a1T23gOMUJrI+uSuH0VRDStOiUVSjBTRDVBVag= +github.com/shurcooL/home v0.0.0-20181020052607-80b7ffcb30f9/go.mod h1:+rgNQw2P9ARFAs37qieuu7ohDNQ3gds9msbT2yn85sg= +github.com/shurcooL/htmlg v0.0.0-20170918183704-d01228ac9e50/go.mod h1:zPn1wHpTIePGnXSHpsVPWEktKXHr6+SS6x/IKRb7cpw= +github.com/shurcooL/httperror v0.0.0-20170206035902-86b7830d14cc/go.mod h1:aYMfkZ6DWSJPJ6c4Wwz3QtW22G7mf/PEgaB9k/ik5+Y= +github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= +github.com/shurcooL/httpgzip v0.0.0-20180522190206-b1c53ac65af9/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q= +github.com/shurcooL/issues v0.0.0-20181008053335-6292fdc1e191/go.mod h1:e2qWDig5bLteJ4fwvDAc2NHzqFEthkqn7aOZAOpj+PQ= +github.com/shurcooL/issuesapp v0.0.0-20180602232740-048589ce2241/go.mod h1:NPpHK2TI7iSaM0buivtFUc9offApnI0Alt/K8hcHy0I= +github.com/shurcooL/notifications v0.0.0-20181007000457-627ab5aea122/go.mod h1:b5uSkrEVM1jQUspwbixRBhaIjIzL2xazXp6kntxYle0= +github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5JD4R5UVWDucdOPg1g2fqQRq78IQa9zlOV1vpQ= +github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk= +github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4= +github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= +github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= +github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/tatsushid/go-prettytable v0.0.0-20141013043238-ed2d14c29939 h1:BhIUXV2ySTLrKgh/Hnts+QTQlIbWtomXt3LMdzME0A0= github.com/tatsushid/go-prettytable v0.0.0-20141013043238-ed2d14c29939/go.mod h1:omGxs4/6hNjxPKUTjmaNkPzehSnNJOJN6pMEbrlYIT4= +github.com/ulikunitz/xz v0.5.5 h1:pFrO0lVpTBXLpYw+pnLj6TbvHuyjXMfjGeCwSqCVwok= +github.com/ulikunitz/xz v0.5.5/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= github.com/urfave/cli v0.0.0-20160620154522-6011f165dc28 h1:Yf7/DcGM+61oLBGXQV2Q+ztEGBcOe3EUnbKSOn4fQuE= github.com/urfave/cli v0.0.0-20160620154522-6011f165dc28/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +go.opencensus.io v0.18.0 h1:Mk5rgZcggtbvtAun5aJzAtjKKN/t0R3jJPlWILlv938= +go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.8.0 h1:r6Za1Rii8+EGOYRDLvpooNOF6kP3iyDnkpzbw67gCQ8= go.uber.org/zap v1.8.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= +golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= golang.org/x/crypto v0.0.0-20180403160946-b2aa35443fbc h1:Kx1Ke+iCR1aDjbWXgmEQGFxoHtNL49aRZGV7/+jJ41Y= golang.org/x/crypto v0.0.0-20180403160946-b2aa35443fbc/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16 h1:y6ce7gCWtnH+m3dCjzQ1PCuwl28DDIc3VNnvY29DlIA= +golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a h1:gOpx8G595UYyvj8UK4+OFyY4rx037g3fmfhe5SasG3U= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890 h1:uESlIz09WIHT2I+pasSXcpLYqYK8wHcdCetU3VuMBJE= +golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181213200352-4d1cda033e06/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.1.0 h1:K6z2u68e86TPdSdefXdzvXgR1zEMa+459vBSfWYAZkI= +google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= +google.golang.org/genproto v0.0.0-20190201180003-4b09977fb922 h1:mBVYJnbrXLA/ZCBTCe7PtEgAUP+1bg92qTaFoPHdz+8= +google.golang.org/genproto v0.0.0-20190201180003-4b09977fb922/go.mod h1:L3J43x8/uS+qIUoksaLKe6OS3nUKxOKuIFz1sl2/jx4= +google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= +google.golang.org/grpc v1.17.0 h1:TRJYBgMclJvGYn2rIMjj+h9KtMt5r1Ij7ODVRIZkwhk= +google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.27/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= +honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= +sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= diff --git a/pkg/app/app.go b/pkg/app/app.go index 924f7009e..15116e594 100644 --- a/pkg/app/app.go +++ b/pkg/app/app.go @@ -3,6 +3,7 @@ package app import ( "fmt" "github.com/roboll/helmfile/pkg/helmexec" + "github.com/roboll/helmfile/pkg/remote" "github.com/roboll/helmfile/pkg/state" "io/ioutil" "log" @@ -42,6 +43,8 @@ type App struct { getwd func() (string, error) chdir func(string) error + + remote *remote.Remote } func New(conf ConfigProvider) *App { @@ -302,6 +305,14 @@ func (a *App) visitStates(fileOrDir string, defOpts LoadOpts, converge func(*sta } else { optsForNestedState.Selectors = m.Selectors } + + path, err := a.remote.Locate(m.Path) + if err != nil { + return appError(fmt.Sprintf("in .helmfiles[%d]", i), err) + } + + m.Path = path + if err := a.visitStates(m.Path, optsForNestedState, converge); err != nil { switch err.(type) { case *NoMatchingHelmfileError: @@ -373,6 +384,26 @@ func (a *App) VisitDesiredStatesWithReleasesFiltered(fileOrDir string, converge opts.Environment.OverrideValues = envvals } + var dir string + if a.directoryExistsAt(fileOrDir) { + dir = fileOrDir + } else { + dir = filepath.Dir(fileOrDir) + } + + getter := &remote.GoGetter{Logger: a.Logger} + + remote := &remote.Remote{ + Logger: a.Logger, + Home: dir, + Getter: getter, + ReadFile: a.readFile, + DirExists: a.directoryExistsAt, + FileExists: a.fileExistsAt, + } + + a.remote = remote + err := a.visitStates(fileOrDir, opts, func(st *state.HelmState, helm helmexec.Interface) (bool, []error) { if len(st.Selectors) > 0 { err := st.FilterReleases() @@ -388,6 +419,7 @@ func (a *App) VisitDesiredStatesWithReleasesFiltered(fileOrDir string, converge type Key struct { TillerNamespace, Name string } + releaseNameCounts := map[Key]int{} for _, r := range st.Releases { tillerNamespace := st.HelmDefaults.TillerNamespace diff --git a/pkg/app/app_test.go b/pkg/app/app_test.go index 6e55bcf42..e2eff266a 100644 --- a/pkg/app/app_test.go +++ b/pkg/app/app_test.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/roboll/helmfile/pkg/helmexec" "github.com/roboll/helmfile/pkg/state" + "github.com/roboll/helmfile/pkg/testhelper" "os" "path/filepath" "reflect" @@ -13,11 +14,11 @@ import ( ) func appWithFs(app *App, files map[string]string) *App { - fs := state.NewTestFs(files) + fs := testhelper.NewTestFs(files) return injectFs(app, fs) } -func injectFs(app *App, fs *state.TestFs) *App { +func injectFs(app *App, fs *testhelper.TestFs) *App { app.readFile = fs.ReadFile app.glob = fs.Glob app.abs = fs.Abs @@ -52,7 +53,7 @@ releases: chart: stable/grafana `, } - fs := state.NewTestFs(files) + fs := testhelper.NewTestFs(files) fs.GlobFixtures["/path/to/helmfile.d/a*.yaml"] = []string{"/path/to/helmfile.d/a2.yaml", "/path/to/helmfile.d/a1.yaml"} app := &App{ KubeContext: "default", @@ -98,7 +99,7 @@ BAR: 2 BAZ: 4 `, } - fs := state.NewTestFs(files) + fs := testhelper.NewTestFs(files) fs.GlobFixtures["/path/to/env.*.yaml"] = []string{"/path/to/env.2.yaml", "/path/to/env.1.yaml"} app := &App{ KubeContext: "default", @@ -137,7 +138,7 @@ releases: chart: stable/zipkin `, } - fs := state.NewTestFs(files) + fs := testhelper.NewTestFs(files) app := &App{ KubeContext: "default", Logger: helmexec.NewLogger(os.Stderr, "debug"), @@ -190,7 +191,7 @@ releases: chart: stable/zipkin `, testcase.handler, testcase.filePattern), } - fs := state.NewTestFs(files) + fs := testhelper.NewTestFs(files) app := &App{ KubeContext: "default", Logger: helmexec.NewLogger(os.Stderr, "debug"), @@ -251,7 +252,7 @@ releases: } for _, testcase := range testcases { - fs := state.NewTestFs(files) + fs := testhelper.NewTestFs(files) fs.GlobFixtures["/path/to/helmfile.d/a*.yaml"] = []string{"/path/to/helmfile.d/a2.yaml", "/path/to/helmfile.d/a1.yaml"} app := &App{ KubeContext: "default", @@ -1077,7 +1078,7 @@ releases: stage: post <<: *default ` - testFs := state.NewTestFs(map[string]string{ + testFs := testhelper.NewTestFs(map[string]string{ yamlFile: yamlContent, "/path/to/base.yaml": `environments: default: @@ -1158,7 +1159,7 @@ releases: stage: post <<: *default ` - testFs := state.NewTestFs(map[string]string{ + testFs := testhelper.NewTestFs(map[string]string{ yamlFile: yamlContent, "/path/to/base.yaml": `environments: default: @@ -1235,7 +1236,7 @@ releases: - name: myrelease0 chart: mychart0 ` - testFs := state.NewTestFs(map[string]string{ + testFs := testhelper.NewTestFs(map[string]string{ yamlFile: yamlContent, "/path/to/base.yaml": `environments: default: @@ -1295,7 +1296,7 @@ releases: - name: myrelease0 chart: mychart0 ` - testFs := state.NewTestFs(map[string]string{ + testFs := testhelper.NewTestFs(map[string]string{ yamlFile: yamlContent, "/path/to/base.yaml": `environments: default: @@ -1372,7 +1373,7 @@ releases: stage: post <<: *default ` - testFs := state.NewTestFs(map[string]string{ + testFs := testhelper.NewTestFs(map[string]string{ yamlFile: yamlContent, "/path/to/base.yaml": `environments: test: @@ -1458,7 +1459,7 @@ releases: chart: mychart3 <<: *default ` - testFs := state.NewTestFs(map[string]string{ + testFs := testhelper.NewTestFs(map[string]string{ yamlFile: yamlContent, "/path/to/yaml/templates.yaml": `templates: default: &default @@ -1515,7 +1516,7 @@ releases: - name: {{ .Environment.Values.foo | quote }} chart: {{ .Environment.Values.bar | quote }} ` - testFs := state.NewTestFs(map[string]string{ + testFs := testhelper.NewTestFs(map[string]string{ statePath: stateContent, "/path/to/1.yaml": `bar: ["bar"]`, "/path/to/2.yaml": `bar: ["BAR"]`, @@ -1568,7 +1569,7 @@ releases: - name: {{ .Environment.Values.foo | quote }} chart: {{ .Environment.Values.bar | quote }} ` - testFs := state.NewTestFs(map[string]string{ + testFs := testhelper.NewTestFs(map[string]string{ statePath: stateContent, "/path/to/1.yaml": `bar: ["bar"]`, "/path/to/2.yaml": `bar: ["BAR"]`, @@ -1653,7 +1654,7 @@ releases: tc := testcases[i] statePath := "/path/to/helmfile.yaml" stateContent := fmt.Sprintf(tc.state, tc.expr) - testFs := state.NewTestFs(map[string]string{ + testFs := testhelper.NewTestFs(map[string]string{ statePath: stateContent, "/path/to/1.yaml": `foo: FOO`, "/path/to/2.yaml": `bar: { "baz": "BAZ" } diff --git a/pkg/app/two_pass_renderer_test.go b/pkg/app/two_pass_renderer_test.go index 701677202..94a64572e 100644 --- a/pkg/app/two_pass_renderer_test.go +++ b/pkg/app/two_pass_renderer_test.go @@ -3,6 +3,7 @@ package app import ( "github.com/roboll/helmfile/pkg/helmexec" "github.com/roboll/helmfile/pkg/state" + "github.com/roboll/helmfile/pkg/testhelper" "os" "strings" "testing" @@ -10,8 +11,8 @@ import ( "gopkg.in/yaml.v2" ) -func makeLoader(files map[string]string, env string) (*desiredStateLoader, *state.TestFs) { - testfs := state.NewTestFs(files) +func makeLoader(files map[string]string, env string) (*desiredStateLoader, *testhelper.TestFs) { + testfs := testhelper.NewTestFs(files) return &desiredStateLoader{ env: env, namespace: "namespace", diff --git a/pkg/remote/remote.go b/pkg/remote/remote.go new file mode 100644 index 000000000..2d42ee1af --- /dev/null +++ b/pkg/remote/remote.go @@ -0,0 +1,242 @@ +package remote + +import ( + "context" + "encoding/json" + "fmt" + "github.com/hashicorp/go-getter" + "github.com/hashicorp/go-getter/helper/url" + "go.uber.org/zap" + "gopkg.in/yaml.v2" + "path/filepath" + "strings" +) + +const DefaultCacheDir = ".helmfile/cache" + +type Remote struct { + Logger *zap.SugaredLogger + + // Home is the home directory for helmfile. Usually this points to $HOME of the user running helmfile. + // Helmfile saves fetched remote files into .helmfile/cache under home + Home string + + // Getter is the underlying implementation of getter used for fetching remote files + Getter Getter + + // ReadFile is the implementation of the file reader that reads a local file from the specified path. + // Inject any implementation of your choice, like an im-memory impl for testing, ioutil.ReadFile for the real-world use. + ReadFile func(string) ([]byte, error) + DirExists func(string) bool + FileExists func(string) bool +} + +func (r *Remote) Unmarshal(src string, dst interface{}) error { + bytes, err := r.GetBytes(src) + if err != nil { + return err + } + + strs := strings.Split(src, "/") + file := strs[len(strs)-1] + ext := filepath.Ext(file) + + { + r.Logger.Debugf("unmarshalling %s", string(bytes)) + + var err error + switch ext { + case "json": + err = json.Unmarshal(bytes, dst) + default: + err = yaml.Unmarshal(bytes, dst) + } + + r.Logger.Debugf("unmarshalled to %v", dst) + + if err != nil { + return err + } + } + + return nil +} + +func (r *Remote) GetBytes(goGetterSrc string) ([]byte, error) { + f, err := r.Fetch(goGetterSrc) + if err != nil { + return nil, err + } + + bytes, err := r.ReadFile(f) + if err != nil { + return nil, fmt.Errorf("read file: %v", err) + } + + return bytes, nil +} + +// Locate takes an URL to a remote file or a path to a local file. +// If the argument was an URL, it fetches the remote directory contained within the URL, +// and returns the path to the file in the fetched directory +func (r *Remote) Locate(urlOrPath string) (string, error) { + fetched, err := r.Fetch(urlOrPath) + if err != nil { + switch err.(type) { + case InvalidURLError: + return urlOrPath, nil + } + return "", err + } + return fetched, nil +} + +type InvalidURLError struct { + err string +} + +func (e InvalidURLError) Error() string { + return e.err +} + +type Source struct { + Getter, Scheme, Host, Dir, File, RawQuery string +} + +func IsRemote(goGetterSrc string) bool { + if _, err := Parse(goGetterSrc); err != nil { + return false + } + return true +} + +func Parse(goGetterSrc string) (*Source, error) { + items := strings.Split(goGetterSrc, "::") + var getter string + switch len(items) { + case 2: + getter = items[0] + goGetterSrc = items[1] + } + + u, err := url.Parse(goGetterSrc) + if err != nil { + return nil, InvalidURLError{err: fmt.Sprintf("parse url: %v", err)} + } + + if u.Scheme == "" { + return nil, InvalidURLError{err: fmt.Sprintf("parse url: missing scheme - probably this is a local file path? %s", goGetterSrc)} + } + + pathComponents := strings.Split(u.Path, "@") + if len(pathComponents) != 2 { + return nil, fmt.Errorf("invalid src format: it must be `[::]:///@?key1=val1&key2=val2: got %s", goGetterSrc) + } + + return &Source{ + Getter: getter, + Scheme: u.Scheme, + Host: u.Host, + Dir: pathComponents[0], + File: pathComponents[1], + RawQuery: u.RawQuery, + }, nil +} + +func (r *Remote) Fetch(goGetterSrc string) (string, error) { + u, err := Parse(goGetterSrc) + if err != nil { + return "", err + } + + srcDir := fmt.Sprintf("%s://%s%s", u.Scheme, u.Host, u.Dir) + file := u.File + + r.Logger.Debugf("getter: %s", u.Getter) + r.Logger.Debugf("scheme: %s", u.Scheme) + r.Logger.Debugf("host: %s", u.Host) + r.Logger.Debugf("dir: %s", u.Dir) + r.Logger.Debugf("file: %s", u.File) + + // This should be shared across variant commands, so that they can share cache for the shared imports + cacheBaseDir := DefaultCacheDir + + query := u.RawQuery + + var cacheKey string + replacer := strings.NewReplacer(":", "", "//", "_", "/", "_", ".", "_") + dirKey := replacer.Replace(srcDir) + if len(query) > 0 { + paramsKey := strings.Replace(query, "&", "_", -1) + cacheKey = fmt.Sprintf("%s.%s", dirKey, paramsKey) + } else { + cacheKey = dirKey + } + + cached := false + + getterDst := filepath.Join(cacheBaseDir, cacheKey) + + cacheDirPath := filepath.Join(r.Home, getterDst) + + { + if r.FileExists(cacheDirPath) { + return "", fmt.Errorf("%s is not directory. please remove it so that variant could use it for dependency caching", getterDst) + } + + if r.DirExists(cacheDirPath) { + cached = true + } + } + + if !cached { + var getterSrc string + + if len(query) == 0 { + getterSrc = srcDir + } else { + getterSrc = strings.Join([]string{srcDir, query}, "?") + } + + if u.Getter != "" { + getterSrc = u.Getter + "::" + getterSrc + } + + r.Logger.Debugf("downloading %s to %s", getterSrc, getterDst) + + if err := r.Getter.Get(r.Home, getterSrc, getterDst); err != nil { + return "", err + } + } + + return filepath.Join(cacheDirPath, file), nil +} + +type Getter interface { + Get(wd, src, dst string) error +} + +type GoGetter struct { + Logger *zap.SugaredLogger +} + +func (g *GoGetter) Get(wd, src, dst string) error { + ctx := context.Background() + + get := &getter.Client{ + Ctx: ctx, + Src: src, + Dst: dst, + Pwd: wd, + Mode: getter.ClientModeDir, + Options: []getter.ClientOption{}, + } + + g.Logger.Debugf("client: %+v", *get) + + if err := get.Get(); err != nil { + return fmt.Errorf("get: %v", err) + } + + return nil +} diff --git a/pkg/remote/remote_test.go b/pkg/remote/remote_test.go new file mode 100644 index 000000000..86e5de3d7 --- /dev/null +++ b/pkg/remote/remote_test.go @@ -0,0 +1,94 @@ +package remote + +import ( + "fmt" + "github.com/roboll/helmfile/pkg/helmexec" + "github.com/roboll/helmfile/pkg/testhelper" + "os" + "testing" +) + +func TestRemote(t *testing.T) { + cleanfs := map[string]string{ + "path/to/home": "", + } + cachefs := map[string]string{ + "path/to/home/.helmfile/cache/https_github_com_cloudposse_helmfiles_git.ref=0.40.0/releases/kiam.yaml": "foo: bar", + } + + type testcase struct { + files map[string]string + expectCacheHit bool + } + + testcases := []testcase{ + {files: cleanfs, expectCacheHit: false}, + {files: cachefs, expectCacheHit: true}, + } + + for i := range testcases { + testcase := testcases[i] + + t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) { + testfs := testhelper.NewTestFs(testcase.files) + + hit := true + + get := func(wd, src, dst string) error { + if wd != "path/to/home" { + return fmt.Errorf("unexpected wd: %s", wd) + } + if src != "git::https://github.com/cloudposse/helmfiles.git?ref=0.40.0" { + return fmt.Errorf("unexpected src: %s", src) + } + + hit = false + + return nil + } + + getter := &testGetter{ + get: get, + } + remote := &Remote{ + Logger: helmexec.NewLogger(os.Stderr, "debug"), + Home: "path/to/home", + Getter: getter, + ReadFile: testfs.ReadFile, + FileExists: testfs.FileExistsAt, + DirExists: testfs.DirectoryExistsAt, + } + + // FYI, go-getter in the `dir` mode accepts URL like the below. So helmfile expects URLs similar to it: + // go-getter -mode dir git::https://github.com/cloudposse/helmfiles.git?ref=0.40.0 gettertest1/b + + // We use `@` to separate dir and the file path. This is a good idea borrowed from helm-git: + // https://github.com/aslafy-z/helm-git + + url := "git::https://github.com/cloudposse/helmfiles.git@releases/kiam.yaml?ref=0.40.0" + file, err := remote.Locate(url) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if file != "path/to/home/.helmfile/cache/https_github_com_cloudposse_helmfiles_git.ref=0.40.0/releases/kiam.yaml" { + t.Errorf("unexpected file located: %s", file) + } + + if testcase.expectCacheHit && !hit { + t.Errorf("unexpected result: unexpected cache miss") + } + if !testcase.expectCacheHit && hit { + t.Errorf("unexpected result: unexpected cache hit") + } + }) + } +} + +type testGetter struct { + get func(wd, src, dst string) error +} + +func (t *testGetter) Get(wd, src, dst string) error { + return t.get(wd, src, dst) +} diff --git a/pkg/state/create_test.go b/pkg/state/create_test.go index 9331b24bd..76863a3c6 100644 --- a/pkg/state/create_test.go +++ b/pkg/state/create_test.go @@ -1,6 +1,7 @@ package state import ( + "github.com/roboll/helmfile/pkg/testhelper" "go.uber.org/zap" "io/ioutil" "path/filepath" @@ -98,7 +99,7 @@ bar: {{ readFile "bar.txt" }} expectedValues := `env: production` - testFs := NewTestFs(map[string]string{ + testFs := testhelper.NewTestFs(map[string]string{ fooYamlFile: string(fooYamlContent), barYamlFile: string(barYamlContent), barTextFile: string(barTextContent), diff --git a/pkg/state/state.go b/pkg/state/state.go index 8c2cf0ed4..1da68bb95 100644 --- a/pkg/state/state.go +++ b/pkg/state/state.go @@ -6,6 +6,7 @@ import ( "github.com/roboll/helmfile/pkg/environment" "github.com/roboll/helmfile/pkg/event" "github.com/roboll/helmfile/pkg/helmexec" + "github.com/roboll/helmfile/pkg/remote" "github.com/roboll/helmfile/pkg/tmpl" "io/ioutil" "os" @@ -1262,6 +1263,11 @@ func (st *HelmState) storage() *Storage { func (st *HelmState) ExpandedHelmfiles() ([]SubHelmfileSpec, error) { helmfiles := []SubHelmfileSpec{} for _, hf := range st.Helmfiles { + if remote.IsRemote(hf.Path) { + helmfiles = append(helmfiles, hf) + continue + } + matches, err := st.storage().ExpandPaths(hf.Path) if err != nil { return nil, err diff --git a/pkg/state/state_test.go b/pkg/state/state_test.go index 24bc0a5c3..ced5f42f1 100644 --- a/pkg/state/state_test.go +++ b/pkg/state/state_test.go @@ -2,6 +2,7 @@ package state import ( "github.com/roboll/helmfile/pkg/helmexec" + "github.com/roboll/helmfile/pkg/testhelper" "io/ioutil" "os" "path/filepath" @@ -16,7 +17,7 @@ import ( var logger = helmexec.NewLogger(os.Stdout, "warn") -func injectFs(st *HelmState, fs *TestFs) *HelmState { +func injectFs(st *HelmState, fs *testhelper.TestFs) *HelmState { st.glob = fs.Glob st.readFile = fs.ReadFile st.fileExists = fs.FileExists @@ -1035,7 +1036,7 @@ func TestHelmState_SyncReleases_MissingValuesFileForUndesiredRelease(t *testing. Releases: []ReleaseSpec{tt.release}, logger: logger, } - fs := NewTestFs(map[string]string{}) + fs := testhelper.NewTestFs(map[string]string{}) state = injectFs(state, fs) helm := &mockHelmExec{ lists: map[listKey]string{}, @@ -1434,7 +1435,7 @@ func TestHelmState_SyncReleasesCleanup(t *testing.T) { return nil }, } - testfs := NewTestFs(map[string]string{ + testfs := testhelper.NewTestFs(map[string]string{ "/path/to/someFile": `foo: FOO`, }) state = injectFs(state, testfs) @@ -1517,7 +1518,7 @@ func TestHelmState_DiffReleasesCleanup(t *testing.T) { return nil }, } - testfs := NewTestFs(map[string]string{ + testfs := testhelper.NewTestFs(map[string]string{ "/path/to/someFile": `foo: bar `, }) diff --git a/pkg/state/testfs.go b/pkg/testhelper/testfs.go similarity index 93% rename from pkg/state/testfs.go rename to pkg/testhelper/testfs.go index ed6c53809..bc99fdbf3 100644 --- a/pkg/state/testfs.go +++ b/pkg/testhelper/testfs.go @@ -1,7 +1,8 @@ -package state +package testhelper import ( "fmt" + "os" "path/filepath" "strings" ) @@ -20,8 +21,10 @@ type TestFs struct { func NewTestFs(files map[string]string) *TestFs { dirs := map[string]bool{} for abs, _ := range files { - d := filepath.Dir(abs) - dirs[d] = true + for d := filepath.Dir(abs); !dirs[d]; d = filepath.Dir(d) { + dirs[d] = true + fmt.Fprintf(os.Stderr, "testfs: recognized dir: %s\n", d) + } } return &TestFs{ Cwd: "/path/to",