Add support for providers and an Artifactory backend

README.md

@@ -4,8 +4,9 @@
 
 ## Description
 
-Anthology is a reimplementation of the Terraform Registry API, intended to be used when your modules can't, shouldn't
-or don't need to be public. For all means and purposes it works in the same way as the [public registry][terraform-registry].
+Anthology is a reimplementation of the Terraform Registry API, intended to be used when your modules or providers can't,
+shouldn't or don't need to be public. For all means and purposes it works in the same way as the
+[public registry][terraform-registry].
 
 
 ## How to use
@@ -15,9 +16,9 @@ or don't need to be public. For all means and purposes it works in the same way
 Every release is automatically published to the [Docker Hub][docker-hub]. You can set commandline parameters by
 overriding the command.
 
-__running on port `80`, using `my-module-bucket` for storage:__
+__running on port `80`, using `my-bucket` for storage:__
 
-`docker run -p 80:80 erikvanbrakel/anthology --port=80 --backend=s3 --s3.bucket=my-module-bucket`
+`docker run -p 80:80 erikvanbrakel/anthology --port=80 --backend=s3 --s3.bucket=my-bucket`
 
 __using docker-compose__
 ```yaml
@@ -26,7 +27,7 @@ version: '2.1'
 services:
 
   registry:
-    command: --port=80 --backend=s3 --s3.bucket=my-module-bucket
+    command: --port=80 --backend=s3 --s3.bucket=my-bucket
     build: erikvanbrakel/anthology:latest
     ports:
       - 80:80
@@ -41,7 +42,7 @@ module "anthology" {
   source  = "erikvanbrakel/anthology/aws"
   version = "0.0.2"
 
-  storage_bucket = "this-bucket-stores-my-modules"
+  storage_bucket = "this-bucket-stores-my-objects"
   tld            = "example.com"                   # the registry will be hosted at registry.example.com
 }
 
@@ -64,16 +65,38 @@ are aware of the cost before provisioning!
 | --ssl.key             | Path to the server certificate    | Any valid path           |         |
 
 ### Filesystem backend
-| Parameter             | Description                       | Allowed                  | Default |
-| --------------------- | --------------------------------- | ------------------------ | ------- |
-| --filesystem.basepath | Base path for module storage      | Any valid path           |         |
+| Parameter                 | Description                       | Allowed                  | Default |
+| ------------------------- | --------------------------------- | ------------------------ | ------- |
+| --filesystem.basepath     | Base path for module storage      | Any valid path           |         |
+| --filesystem.providerpath | Base path for provider storage    | Any valid path           |         |
 
 ### S3 backend
 | Parameter             | Description                       | Allowed                    | Default |
 | --------------------- | --------------------------------- | -------------------------- | ------- |
 | --s3.bucket           | Name of the S3 bucket for storage | Any valid s3 bucket name   |         |
 | --s3.endpoint         | Alternative S3 endpoint           | http[s]://[hostname]:[port]|         |
 
+### Artifactory backend
+| Parameter                 | Description                               | Allowed                   | Default |
+| ------------------------- | ----------------------------------------- | ------------------------- | ------- |
+| --artifactory.moduleurl   | Artifactory API URL to store modules      | Any valid Artifactory URL |         |
+| --artifactory.providerurl | Artifactory API URL to store providers    | Any valid Artifactory URL |         |
+
 [terraform-registry]: https://registry.terraform.io/
 [anthology-module]: https://registry.terraform.io/modules/erikvanbrakel/anthology/aws/
 [docker-hub]: https://hub.docker.com/r/erikvanbrakel/anthology/
+
+## Provider file formats and directory struction
+
+The directory structure and file naming conventions must follow the Hashicorp standards for providers, as documented at https://www.terraform.io/docs/internals/provider-registry-protocol.html.
+
+For example, for a provider named `myprovider` in the `example` namespace, the expected setup is like follows (with deviations for the OS and Arch that the provider is built for):
+
+/providers/example/myprovider/0.0.1/terraform-provider-myprovider_0.0.1_linux_amd64.zip
+/providers/example/myprovider/0.0.1/terraform-provider-myprovider_0.0.1_darwin_amd64.zip
+/providers/example/myprovider/0.0.1/terraform-provider-myprovider_0.0.1_windows_amd64.zip
+/providers/example/myprovider/0.0.1/terraform-provider-myprovider_0.0.1_SHA256SUMS.sig
+/providers/example/myprovider/0.0.1/terraform-provider-myprovider_0.0.1_SHA256SUMS
+/providers/example/myprovider/0.0.1/terraform-provider-myprovider_0.0.1.gpg
+
+The SHA245SUMS and SHA256SUMS.sig files are as documented by Hashicorp.  The .gpg file contains an ASCII armored _public_ GPG key used to sign the SHA256SUMS file.

api/v1/api_test.go

@@ -39,7 +39,7 @@ func newRouter() *routing.Router {
 	return router
 }
 
-func runAPITests(t *testing.T, dataset []testModule, tests []apiTestCase) {
+func runModuleAPITests(t *testing.T, dataset []testModule, tests []apiTestCase) {
 	for _, test := range tests {
 		t.Run(test.tag, func(t *testing.T) {
 			r := registry.NewFakeRegistry()
@@ -70,10 +70,57 @@ func runAPITests(t *testing.T, dataset []testModule, tests []apiTestCase) {
 	}
 }
 
+func runProviderAPITests(t *testing.T, dataset []testProvider, tests []apiTestCase) {
+	for _, test := range tests {
+		t.Run(test.tag, func(t *testing.T) {
+			r := registry.NewFakeRegistry()
+
+			for _, p := range dataset {
+				r.PublishProvider(p.namespace, p.name, p.version, p.os, p.arch, bytes.NewBuffer(p.data))
+			}
+
+			router := newRouter()
+			v1.ServeProviderResource(&router.RouteGroup, services.NewProviderService(r))
+			server := httptest.NewServer(router)
+			defer server.Close()
+
+			e := httpexpect.New(t, server.URL)
+
+			result := e.Request(test.method, test.url).
+				WithClient(&http.Client{
+					CheckRedirect: func(req *http.Request, via []*http.Request) error {
+						return http.ErrUseLastResponse
+					},
+				}).
+				WithHeader("Content-Type", "application/json").
+				WithBytes([]byte(test.body)).
+				Expect().Status(test.status)
+
+			test.assert(t, result, server)
+		})
+	}
+}
+
+func assertError(error string) func(*testing.T, *httpexpect.Response, *httptest.Server) {
+	return func(t *testing.T, r *httpexpect.Response, server *httptest.Server) {
+		errors := r.JSON().Object().Value("errors").Array()
+		errors.Contains(error)
+	}
+}
+
 type testModule struct {
 	namespace string
 	name      string
 	provider  string
 	version   string
 	data      []byte
 }
+
+type testProvider struct {
+	namespace string
+	name      string
+	version   string
+	os        string
+	arch      string
+	data      []byte
+}

api/v1/base.go

@@ -0,0 +1,14 @@
+package v1
+
+type apiError struct {
+	Errors []string `json:"errors"`
+}
+
+type PaginationInfo struct {
+	Limit          int    `json:"limit"`
+	PreviousOffset int    `json:"previous_offset"`
+	PreviousUrl    int    `json:"previous_url"`
+	CurrentOffset  int    `json:"current_offset"`
+	NextOffset     int    `json:"next_offset"`
+	NextUrl        string `json:"next_url"`
+}

api/v1/module.go

@@ -24,10 +24,6 @@ type (
 	moduleResource struct {
 		service moduleService
 	}
-
-	apiError struct {
-		Errors []string `json:"errors"`
-	}
 )
 
 func ServeModuleResource(rg *routing.RouteGroup, service moduleService) {
@@ -115,9 +111,9 @@ func (r *moduleResource) query(c *routing.Context) error {
 		return c.Write(apiError{[]string{"not found"}})
 	}
 
-	paginationInfo := getPaginationInfo(c, count)
+	paginationInfo := getModulePaginationInfo(c, count)
 
-	return c.Write(PaginatedList{
+	return c.Write(ModulePaginatedList{
 		PaginationInfo: paginationInfo,
 		Modules:        modules,
 	})
@@ -142,9 +138,9 @@ func (r *moduleResource) queryVersions(c *routing.Context) error {
 	}
 
 	return c.Write(struct {
-		Modules VersionsList `json:"modules"`
+		Modules ModuleVersionsList `json:"modules"`
 	}{
-		VersionsList{
+		ModuleVersionsList{
 
 			{
 				Source:   fmt.Sprintf("%s/%s/%s", namespace, name, provider),
@@ -304,18 +300,18 @@ func (r *moduleResource) queryLatest(c *routing.Context) error {
 		v = append(v, value)
 	}
 
-	return c.Write(PaginatedList{
-		PaginationInfo: getPaginationInfo(c, len(v)),
+	return c.Write(ModulePaginatedList{
+		PaginationInfo: getModulePaginationInfo(c, len(v)),
 		Modules:        v,
 	})
 }
 
-type VersionsList []struct {
+type ModuleVersionsList []struct {
 	Source   string          `json:"source"`
 	Versions []models.Module `json:"versions"`
 }
 
-func getPaginationInfo(c *routing.Context, count int) PaginationInfo {
+func getModulePaginationInfo(c *routing.Context, count int) PaginationInfo {
 	limit, _ := strconv.Atoi(c.Query("limit", "10"))
 	offset, _ := strconv.Atoi(c.Query("offset", "0"))
 
@@ -325,16 +321,7 @@ func getPaginationInfo(c *routing.Context, count int) PaginationInfo {
 	}
 }
 
-type PaginatedList struct {
+type ModulePaginatedList struct {
 	PaginationInfo PaginationInfo `json:"meta"`
 	Modules        interface{}    `json:"modules"`
 }
-
-type PaginationInfo struct {
-	Limit          int    `json:"limit"`
-	PreviousOffset int    `json:"previous_offset"`
-	PreviousUrl    int    `json:"previous_url"`
-	CurrentOffset  int    `json:"current_offset"`
-	NextOffset     int    `json:"next_offset"`
-	NextUrl        string `json:"next_url"`
-}

api/v1/module_test.go

@@ -18,7 +18,7 @@ func TestListModule(t *testing.T) {
 		{"namespace2", "module1", "aws", "2.0.0", nil},
 	}
 
-	runAPITests(t, dataset, []apiTestCase{
+	runModuleAPITests(t, dataset, []apiTestCase{
 		{
 			"get all modules",
 			"GET", "/", "",
@@ -66,7 +66,7 @@ func TestListModuleVersions(t *testing.T) {
 		{"namespace2", "module1", "gcp", "1.0.0", nil},
 	}
 
-	runAPITests(t, dataset, []apiTestCase{
+	runModuleAPITests(t, dataset, []apiTestCase{
 		{
 			"list available versions for a specific module",
 			"GET", "/namespace1/module1/aws/versions", "",
@@ -93,14 +93,14 @@ func TestListModuleVersions(t *testing.T) {
 	})
 }
 
-func TestGetDownloadUrl(t *testing.T) {
+func TestGetModuleDownloadUrl(t *testing.T) {
 	dataset := []testModule{
 		{"namespace1", "module1", "aws", "1.0.0", nil},
 		{"namespace1", "module1", "aws", "2.0.0", nil},
 		{"namespace1", "module1", "aws", "3.0.0", nil},
 	}
 
-	runAPITests(t, dataset, []apiTestCase{
+	runModuleAPITests(t, dataset, []apiTestCase{
 		{
 			"download source code for a specific module version",
 			"GET", "/namespace1/module1/aws/1.0.0/download", "",
@@ -132,7 +132,7 @@ func TestGetDownloadUrl(t *testing.T) {
 	})
 }
 
-func TestListLatestVersions(t *testing.T) {
+func TestListModuleLatestVersions(t *testing.T) {
 	dataset := []testModule{
 		{"namespace1", "module1", "aws", "1.0.0", nil},
 		{"namespace1", "module1", "aws", "2.0.0", nil},
@@ -142,7 +142,7 @@ func TestListLatestVersions(t *testing.T) {
 		{"namespace1", "module1", "azure", "6.6.0", nil},
 	}
 
-	runAPITests(t, dataset, []apiTestCase{
+	runModuleAPITests(t, dataset, []apiTestCase{
 		{
 			"list latest version of module for all providers",
 			"GET", "/namespace1/module1", "",
@@ -175,7 +175,7 @@ func TestGetModule(t *testing.T) {
 		{"namespace1", "module1", "azure", "6.6.0", nil},
 	}
 
-	runAPITests(t, dataset, []apiTestCase{
+	runModuleAPITests(t, dataset, []apiTestCase{
 		{
 			"get a specific module",
 			"GET", "/namespace1/module1/gcp/3.0.0", "",
@@ -219,7 +219,7 @@ func TestPublishModule(t *testing.T) {
 
 	moduleData := "some data"
 
-	runAPITests(t, dataset, []apiTestCase{
+	runModuleAPITests(t, dataset, []apiTestCase{
 		{
 			"publish a new module",
 			"POST", "/namespace1/module1/gcp/3.0.0", moduleData,
@@ -235,10 +235,3 @@ func TestPublishModule(t *testing.T) {
 		},
 	})
 }
-
-func assertError(error string) func(*testing.T, *httpexpect.Response, *httptest.Server) {
-	return func(t *testing.T, r *httpexpect.Response, server *httptest.Server) {
-		errors := r.JSON().Object().Value("errors").Array()
-		errors.Contains(error)
-	}
-}