diff --git a/builtin/bins/provider-external/main.go b/builtin/bins/provider-external/main.go new file mode 100644 index 000000000000..15f764bef41d --- /dev/null +++ b/builtin/bins/provider-external/main.go @@ -0,0 +1,12 @@ +package main + +import ( + "github.com/hashicorp/terraform/builtin/providers/external" + "github.com/hashicorp/terraform/plugin" +) + +func main() { + plugin.Serve(&plugin.ServeOpts{ + ProviderFunc: external.Provider, + }) +} diff --git a/builtin/providers/external/data_source.go b/builtin/providers/external/data_source.go new file mode 100644 index 000000000000..f41e86b67fd2 --- /dev/null +++ b/builtin/providers/external/data_source.go @@ -0,0 +1,93 @@ +package external + +import ( + "bytes" + "encoding/json" + "fmt" + "os/exec" + + "github.com/hashicorp/terraform/helper/schema" +) + +func dataSource() *schema.Resource { + return &schema.Resource{ + Read: dataSourceRead, + + Schema: map[string]*schema.Schema{ + "program": &schema.Schema{ + Type: schema.TypeList, + Required: true, + Elem: &schema.Schema{ + Type: schema.TypeString, + }, + }, + + "query": &schema.Schema{ + Type: schema.TypeMap, + Optional: true, + Elem: &schema.Schema{ + Type: schema.TypeString, + }, + }, + + "result": &schema.Schema{ + Type: schema.TypeMap, + Computed: true, + Elem: &schema.Schema{ + Type: schema.TypeString, + }, + }, + }, + } +} + +func dataSourceRead(d *schema.ResourceData, meta interface{}) error { + + programI := d.Get("program").([]interface{}) + query := d.Get("query").(map[string]interface{}) + + // This would be a ValidateFunc if helper/schema allowed these + // to be applied to lists. + if err := validateProgramAttr(programI); err != nil { + return err + } + + program := make([]string, len(programI)) + for i, vI := range programI { + program[i] = vI.(string) + } + + cmd := exec.Command(program[0], program[1:]...) + + queryJson, err := json.Marshal(query) + if err != nil { + // Should never happen, since we know query will always be a map + // from string to string, as guaranteed by d.Get and our schema. + return err + } + + cmd.Stdin = bytes.NewReader(queryJson) + + resultJson, err := cmd.Output() + if err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + if exitErr.Stderr != nil && len(exitErr.Stderr) > 0 { + return fmt.Errorf("failed to execute %q: %s", program[0], string(exitErr.Stderr)) + } + return fmt.Errorf("command %q failed with no error message", program[0]) + } else { + return fmt.Errorf("failed to execute %q: %s", program[0], err) + } + } + + result := map[string]string{} + err = json.Unmarshal(resultJson, &result) + if err != nil { + return fmt.Errorf("command %q produced invalid JSON: %s", program[0], err) + } + + d.Set("result", result) + + d.SetId("-") + return nil +} diff --git a/builtin/providers/external/data_source_test.go b/builtin/providers/external/data_source_test.go new file mode 100644 index 000000000000..dbdf003864a5 --- /dev/null +++ b/builtin/providers/external/data_source_test.go @@ -0,0 +1,124 @@ +package external + +import ( + "fmt" + "os" + "os/exec" + "path" + "regexp" + "testing" + + "github.com/hashicorp/terraform/helper/resource" + "github.com/hashicorp/terraform/terraform" +) + +const testDataSourceConfig_basic = ` +data "external" "test" { + program = ["%s", "cheese"] + + query = { + value = "pizza" + } +} + +output "query_value" { + value = "${data.external.test.result["query_value"]}" +} + +output "argument" { + value = "${data.external.test.result["argument"]}" +} +` + +func TestDataSource_basic(t *testing.T) { + programPath, err := buildDataSourceTestProgram() + if err != nil { + t.Fatal(err) + return + } + + resource.UnitTest(t, resource.TestCase{ + Providers: testProviders, + Steps: []resource.TestStep{ + resource.TestStep{ + Config: fmt.Sprintf(testDataSourceConfig_basic, programPath), + Check: func(s *terraform.State) error { + _, ok := s.RootModule().Resources["data.external.test"] + if !ok { + return fmt.Errorf("missing data resource") + } + + outputs := s.RootModule().Outputs + + if outputs["argument"] == nil { + return fmt.Errorf("missing 'argument' output") + } + if outputs["query_value"] == nil { + return fmt.Errorf("missing 'query_value' output") + } + + if outputs["argument"].Value != "cheese" { + return fmt.Errorf( + "'argument' output is %q; want 'cheese'", + outputs["argument"].Value, + ) + } + if outputs["query_value"].Value != "pizza" { + return fmt.Errorf( + "'query_value' output is %q; want 'pizza'", + outputs["query_value"].Value, + ) + } + + return nil + }, + }, + }, + }) +} + +const testDataSourceConfig_error = ` +data "external" "test" { + program = ["%s"] + + query = { + fail = "true" + } +} +` + +func TestDataSource_error(t *testing.T) { + programPath, err := buildDataSourceTestProgram() + if err != nil { + t.Fatal(err) + return + } + + resource.UnitTest(t, resource.TestCase{ + Providers: testProviders, + Steps: []resource.TestStep{ + resource.TestStep{ + Config: fmt.Sprintf(testDataSourceConfig_error, programPath), + ExpectError: regexp.MustCompile("I was asked to fail"), + }, + }, + }) +} + +func buildDataSourceTestProgram() (string, error) { + // We have a simple Go program that we use as a stub for testing. + cmd := exec.Command( + "go", "install", + "github.com/hashicorp/terraform/builtin/providers/external/test-programs/tf-acc-external-data-source", + ) + err := cmd.Run() + + if err != nil { + return "", fmt.Errorf("failed to build test stub program: %s", err) + } + + programPath := path.Join( + os.Getenv("GOPATH"), "bin", "tf-acc-external-data-source", + ) + return programPath, nil +} diff --git a/builtin/providers/external/provider.go b/builtin/providers/external/provider.go new file mode 100644 index 000000000000..24a72ad98f09 --- /dev/null +++ b/builtin/providers/external/provider.go @@ -0,0 +1,15 @@ +package external + +import ( + "github.com/hashicorp/terraform/helper/schema" + "github.com/hashicorp/terraform/terraform" +) + +func Provider() terraform.ResourceProvider { + return &schema.Provider{ + DataSourcesMap: map[string]*schema.Resource{ + "external": dataSource(), + }, + ResourcesMap: map[string]*schema.Resource{}, + } +} diff --git a/builtin/providers/external/provider_test.go b/builtin/providers/external/provider_test.go new file mode 100644 index 000000000000..b5afda93cc51 --- /dev/null +++ b/builtin/providers/external/provider_test.go @@ -0,0 +1,18 @@ +package external + +import ( + "testing" + + "github.com/hashicorp/terraform/helper/schema" + "github.com/hashicorp/terraform/terraform" +) + +func TestProvider(t *testing.T) { + if err := Provider().(*schema.Provider).InternalValidate(); err != nil { + t.Fatalf("err: %s", err) + } +} + +var testProviders = map[string]terraform.ResourceProvider{ + "external": Provider(), +} diff --git a/builtin/providers/external/test-programs/tf-acc-external-data-source/main.go b/builtin/providers/external/test-programs/tf-acc-external-data-source/main.go new file mode 100644 index 000000000000..f495cc249158 --- /dev/null +++ b/builtin/providers/external/test-programs/tf-acc-external-data-source/main.go @@ -0,0 +1,50 @@ +package main + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" +) + +// This is a minimal implementation of the external data source protocol +// intended only for use in the provider acceptance tests. +// +// In practice it's likely not much harder to just write a real Terraform +// plugin if you're going to be writing your data source in Go anyway; +// this example is just in Go because we want to avoid introducing +// additional language runtimes into the test environment. +func main() { + queryBytes, err := ioutil.ReadAll(os.Stdin) + if err != nil { + panic(err) + } + + var query map[string]string + err = json.Unmarshal(queryBytes, &query) + if err != nil { + panic(err) + } + + if query["fail"] != "" { + fmt.Fprintf(os.Stderr, "I was asked to fail\n") + os.Exit(1) + } + + var result = map[string]string{ + "result": "yes", + "query_value": query["value"], + } + + if len(os.Args) >= 2 { + result["argument"] = os.Args[1] + } + + resultBytes, err := json.Marshal(result) + if err != nil { + panic(err) + } + + os.Stdout.Write(resultBytes) + os.Exit(0) +} diff --git a/builtin/providers/external/util.go b/builtin/providers/external/util.go new file mode 100644 index 000000000000..fd378fb3e1ed --- /dev/null +++ b/builtin/providers/external/util.go @@ -0,0 +1,35 @@ +package external + +import ( + "fmt" + "os/exec" +) + +// validateProgramAttr is a validation function for the "program" attribute we +// accept as input on our resources. +// +// The attribute is assumed to be specified in schema as a list of strings. +func validateProgramAttr(v interface{}) error { + args := v.([]interface{}) + if len(args) < 1 { + return fmt.Errorf("'program' list must contain at least one element") + } + + for i, vI := range args { + if _, ok := vI.(string); !ok { + return fmt.Errorf( + "'program' element %d is %T; a string is required", + i, vI, + ) + } + } + + // first element is assumed to be an executable command, possibly found + // using the PATH environment variable. + _, err := exec.LookPath(args[0].(string)) + if err != nil { + return fmt.Errorf("can't find external program %q", args[0]) + } + + return nil +} diff --git a/command/internal_plugin_list.go b/command/internal_plugin_list.go index 8727578b833b..4b500bada5bb 100644 --- a/command/internal_plugin_list.go +++ b/command/internal_plugin_list.go @@ -24,6 +24,7 @@ import ( dnsimpleprovider "github.com/hashicorp/terraform/builtin/providers/dnsimple" dockerprovider "github.com/hashicorp/terraform/builtin/providers/docker" dynprovider "github.com/hashicorp/terraform/builtin/providers/dyn" + externalprovider "github.com/hashicorp/terraform/builtin/providers/external" fastlyprovider "github.com/hashicorp/terraform/builtin/providers/fastly" githubprovider "github.com/hashicorp/terraform/builtin/providers/github" googleprovider "github.com/hashicorp/terraform/builtin/providers/google" @@ -84,6 +85,7 @@ var InternalProviders = map[string]plugin.ProviderFunc{ "dnsimple": dnsimpleprovider.Provider, "docker": dockerprovider.Provider, "dyn": dynprovider.Provider, + "external": externalprovider.Provider, "fastly": fastlyprovider.Provider, "github": githubprovider.Provider, "google": googleprovider.Provider, diff --git a/terraform/util.go b/terraform/util.go index d1ca197eade4..e1d951c01a67 100644 --- a/terraform/util.go +++ b/terraform/util.go @@ -54,7 +54,10 @@ func resourceProvider(t, alias string) string { idx := strings.IndexRune(t, '_') if idx == -1 { - return "" + // If no underscores, the resource name is assumed to be + // also the provider name, e.g. if the provider exposes + // only a single resource of each type. + return t } return t[:idx] diff --git a/terraform/util_test.go b/terraform/util_test.go index 560c8a87386a..3c95e361a06f 100644 --- a/terraform/util_test.go +++ b/terraform/util_test.go @@ -46,3 +46,53 @@ func TestStrSliceContains(t *testing.T) { t.Fatalf("Bad") } } + +func TestUtilResourceProvider(t *testing.T) { + type testCase struct { + ResourceName string + Alias string + Expected string + } + + tests := []testCase{ + { + // If no alias is provided, the first underscore-separated segment + // is assumed to be the provider name. + ResourceName: "aws_thing", + Alias: "", + Expected: "aws", + }, + { + // If we have more than one underscore then it's the first one that we'll use. + ResourceName: "aws_thingy_thing", + Alias: "", + Expected: "aws", + }, + { + // A provider can export a resource whose name is just the bare provider name, + // e.g. because the provider only has one resource and so any additional + // parts would be redundant. + ResourceName: "external", + Alias: "", + Expected: "external", + }, + { + // Alias always overrides the default extraction of the name + ResourceName: "aws_thing", + Alias: "tls.baz", + Expected: "tls.baz", + }, + } + + for _, test := range tests { + got := resourceProvider(test.ResourceName, test.Alias) + if got != test.Expected { + t.Errorf( + "(%q, %q) produced %q; want %q", + test.ResourceName, test.Alias, + got, + test.Expected, + ) + } + } +} diff --git a/website/source/assets/stylesheets/_docs.scss b/website/source/assets/stylesheets/_docs.scss index 791cc774d76b..f9a85757deee 100755 --- a/website/source/assets/stylesheets/_docs.scss +++ b/website/source/assets/stylesheets/_docs.scss @@ -25,6 +25,7 @@ body.layout-dme, body.layout-dnsimple, body.layout-docker, body.layout-dyn, +body.layout-external, body.layout-github, body.layout-grafana, body.layout-fastly, diff --git a/website/source/docs/providers/external/data_source.html.md b/website/source/docs/providers/external/data_source.html.md new file mode 100644 index 000000000000..ca983d41ea50 --- /dev/null +++ b/website/source/docs/providers/external/data_source.html.md @@ -0,0 +1,116 @@ +--- +layout: "external" +page_title: "External Data Source" +sidebar_current: "docs-external-data-source" +description: |- + Executes an external program that implements a data source. +--- + +# External Data Source + +The `external` data source allows an external program implementing a specific +protocol (defined below) to act as a data source, exposing arbitrary +data for use elsewhere in the Terraform configuration. + +~> **Warning** This mechanism is provided as an "escape hatch" for exceptional +situations where a first-class Terraform provider is not more appropriate. +Its capabilities are limited in comparison to a true data source, and +implementing a data source via an external program is likely to hurt the +portability of your Terraform configuration by creating dependencies on +external programs and libraries that may not be available (or may need to +be used differently) on different operating systems. + +~> **Warning** Terraform Enterprise does not guarantee availability of any +particular language runtimes or external programs beyond standard shell +utilities, so it is not recommended to use this data source within +configurations that are applied within Terraform Enterprise. + +## Example Usage + +``` +data "external" "example" { + program = ["python", "${path.module}/example-data-source.py"] + + query = { + # arbitrary map from strings to strings, passed + # to the external program as the data query. + id = "abc123" + } +} +``` + +## External Program Protocol + +The external program described by the `program` attribute must implement a +specific protocol for interacting with Terraform, as follows. + +The program must read all of the data passed to it on `stdin`, and parse +it as a JSON object. The JSON object contains the contents of the `query` +argument and its values will always be strings. + +The program must then produce a valid JSON object on `stdout`, which will +be used to populate the `result` attribute exported to the rest of the +Terraform configuration. This JSON object must again have all of its +values as strings. On successful completion it must exit with status zero. + +If the program encounters an error and is unable to produce a result, it +must print a human-readable error message (ideally a single line) to `stderr` +and exit with a non-zero status. Any data on `stdout` is ignored if the +program returns a non-zero status. + +All environment variables visible to the Terraform process are passed through +to the child program. + +Terraform expects a data source to have *no observable side-effects*, and will +re-run the program each time the state is refreshed. + +## Argument Reference + +The following arguments are supported: + +* `program` - (Required) A list of strings, whose first element is the program + to run and whose subsequent elements are optional command line arguments + to the program. Terraform does not execute the program through a shell, so + it is not necessary to escape shell metacharacters nor add quotes around + arguments containing spaces. + +* `query` - (Optional) A map of string values to pass to the external program + as the query arguments. If not supplied, the program will recieve an empty + object as its input. + +## Attributes Reference + +The following attributes are exported: + +* `result` - A map of string values returned from the external program. + +## Processing JSON in shell scripts + +Since the external data source protocol uses JSON, it is recommended to use +the utility [`jq`](https://stedolan.github.io/jq/) to translate to and from +JSON in a robust way when implementing a data source in a shell scripting +language. + +The following example shows some input/output boilerplate code for a +data source implemented in bash: + +``` +#!/bin/bash + +# Exit if any of the intermediate steps fail +set -e + +# Extract "foo" and "baz" arguments from the input into +# FOO and BAZ shell variables. +# jq will ensure that the values are properly quoted +# and escaped for consumption by the shell. +eval "$(jq -r '@sh "FOO=\(.foo) BAZ=\(.baz)"')" + +# Placeholder for whatever data-fetching logic your script implements +FOOBAZ="$FOO BAZ" + +# Safely produce a JSON object containing the result value. +# jq will ensure that the value is properly quoted +# and escaped to produce a valid JSON string. +jq -n --arg foobaz "$FOOBAZ" '{"foobaz":$foobaz}' +``` diff --git a/website/source/docs/providers/external/index.html.markdown b/website/source/docs/providers/external/index.html.markdown new file mode 100644 index 000000000000..cd7ee01d973d --- /dev/null +++ b/website/source/docs/providers/external/index.html.markdown @@ -0,0 +1,27 @@ +--- +layout: "external" +page_title: "Provider: External" +sidebar_current: "docs-external-index" +description: |- + The external provider allows external scripts to be integrated with Terraform. +--- + +# External Provider + +`external` is a special provider that exists to provide an interface +between Terraform and external programs. + +Using this provider it is possible to write separate programs that can +participate in the Terraform workflow by implementing a specific protocol. + +This provider is intended to be used for simple situations where you wish +to integrate Terraform with a system for which a first-class provider +doesn't exist. It is not as powerful as a first-class Terraform provider, +so users of this interface should carefully consider the implications +described on each of the child documentation pages (available from the +navigation bar) for each type of object this provider supports. + +~> **Warning** Terraform Enterprise does not guarantee availability of any +particular language runtimes or external programs beyond standard shell +utilities, so it is not recommended to use this provider within configurations +that are applied within Terraform Enterprise. diff --git a/website/source/layouts/docs.erb b/website/source/layouts/docs.erb index 70983fcdf412..ceb9193af5bc 100644 --- a/website/source/layouts/docs.erb +++ b/website/source/layouts/docs.erb @@ -234,6 +234,10 @@ Dyn + > + External + + > GitHub diff --git a/website/source/layouts/external.erb b/website/source/layouts/external.erb new file mode 100644 index 000000000000..d566524fb474 --- /dev/null +++ b/website/source/layouts/external.erb @@ -0,0 +1,22 @@ +<% wrap_layout :inner do %> + <% content_for :sidebar do %> + + <% end %> + + <%= yield %> +<% end %>