v1.0

.travis.yml

@@ -1,7 +1,8 @@
 language: go
 
 go:
-  - 1.8
+  - 1.9
+  - "1.10"
 
 install:
   - go get github.com/Masterminds/glide

Makefile

@@ -1,17 +1,39 @@
-VERSION := `cat VERSION`
+VERSION := `git log -n 1 | grep commit | sed 's/commit //g' | head -n 1`
 LDFLAGS := -X main.version=$(VERSION)
 GO := "$(GOROOT)/bin/go"
 
-all: lint test build
+all: vars formatting lint test build
+
+vars:
+	@echo "====== Makefile internal variables:"
+	@echo "VERSION: '$(VERSION)'"
+	@echo "LDFLAGS: '$(LDFLAGS)'"
+	@echo "GO: '$(GO)'"
+	@echo "======\n\n"
+
+linux: vars formatting lint test
+	GOOS=linux $(GO) build -v -ldflags "$(LDFLAGS)" -tags 'netcgo=1'.
+
+formatting:
+	$(GO) get golang.org/x/tools/cmd/goimports
 
 lint: deps-lint
 	gometalinter ./... \
 	--disable=gotype \
 	--disable=dupl \
+	--disable=gosec \
 	--deadline=600s \
-	--enable=goimports \
+	--disable=goimports \
 	--vendor \
-        --fast
+	--fast
+
+lint-slow: deps-lint
+	gometalinter ./... \
+	--disable=gotype \
+	--disable=dupl \
+	--deadline=600s \
+	--enable=goimports \
+	--vendor
 
 deps:
 	$(GO) get github.com/Masterminds/glide
@@ -21,7 +43,7 @@ deps-lint: deps
 	$(GO) get github.com/alecthomas/gometalinter
 	gometalinter --install
 
-build: deps lint
+build: vars deps lint
         # Enable netcgo, then name resolution will use systems dns caches
 	$(GO) build -v -ldflags "$(LDFLAGS)" -tags 'netcgo=1'.
 

README.md

@@ -15,8 +15,8 @@
 Akubra is a simple solution to keep an independent S3 storages in sync - almost
 realtime, eventually consistent.
 
-Keeping synchronized storage clusters, which handles great volume of new objects
-(about 300k obj/h), is the most efficient by feeding them with all incoming data
+Keeping synchronized storage clusters, which handles great volume of new objects,
+is the most efficient by feeding them with all incoming data
 at once. That's what Akubra does, with a minimum memory and cpu footprint.
 
 Synchronizing S3 storages offline is almost impossible with a high volume traffic.
@@ -37,6 +37,11 @@ It also backtracks to older cluster when requested for not existing object on
 target cluster. This kind of events are logged, so it's possible to rebalance
 clusters in background.
 
+### Multi cloud cost optimization
+While all objects has to be written in every storage from shard, not all storages
+has to be read. With load balancing and storage prioritization akubra will peak
+cheapest one.
+
 ## Build
 
 ### Prerequisites
@@ -93,47 +98,65 @@ the pool and error is logged.
 Configuration is read from a YAML configuration file with the following fields:
 
 ```yaml
-# Listen interface and port e.g. "127.0.0.1:9090", ":80"
-Listen: ":8080"
-# Technical endpoint interface
-TechnicalEndpointListen: ":8071"
-# Technical health check endpoint (for load balancers)
-HealthCheckEndpoint: "/status/ping"
-# Additional not AWS S3 specific headers proxy will add to original request
-AdditionalRequestHeaders:
-    'Cache-Control': "public, s-maxage=600, max-age=600"
-    'X-Akubra-Version': '0.9.26'
-# Additional headers added to backend response
-AdditionalResponseHeaders:
-    'Access-Control-Allow-Origin': "*"
-    'Access-Control-Allow-Credentials': "true"
-    'Access-Control-Allow-Methods': "GET, POST, OPTIONS"
-    'Access-Control-Allow-Headers': "DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type"
-# MaxIdleConns see: https://golang.org/pkg/net/http/#Transport
-# Default 0 (no limit)
-MaxIdleConns: 0
-# MaxIdleConnsPerHost see: https://golang.org/pkg/net/http/#Transport
-# Default 100
-MaxIdleConnsPerHost: 100
-# IdleConnTimeout see: https://golang.org/pkg/net/http/#Transport
-# Default 0 (no limit)
-IdleConnTimeout: 0s
-# ResponseHeaderTimeout see: https://golang.org/pkg/net/http/#Transport
-# Default 5s
-ResponseHeaderTimeout: 5s
-# DisableKeepAlives see: https://golang.org/pkg/net/http/#Transport
-# Default false
-
-DisableKeepAlives: false
-
-# Maximum accepted body size
-BodyMaxSize: "100M"
-# Maximum number of incoming requests to process at once
-MaxConcurrentRequests: 200
-# Backend in maintenance mode. Akubra will skip this endpoint
-
-# MaintainedBackends:
-#  - "http://s3.dc2.internal"
+Service:
+  Server:
+    BodyMaxSize: 100MB
+    MaxConcurrentRequests: 200
+    # Listen interface and port e.g. "0:8000", "localhost:9090", ":80"
+    Listen: ":7082"
+    # Technical endpoint interface
+    TechnicalEndpointListen: ":7005"
+    # Health check endpoint (for load balancers)
+    HealthCheckEndpoint: "/status/ping"
+  Client:
+    # Additional not AWS S3 specific headers proxy will add to original request
+    AdditionalResponseHeaders:
+        'Access-Control-Allow-Origin': "*"
+        'Access-Control-Allow-Credentials': "true"
+        'Access-Control-Allow-Methods': "GET, POST, OPTIONS"
+        'Access-Control-Allow-Headers': "DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,X-CSRFToken"
+        'Cache-Control': "public, s-maxage=600, max-age=600"
+    # Additional headers added to backend response
+    AdditionalRequestHeaders:
+        'Cache-Control': "public, s-maxage=600, max-age=600"
+    # Backends in maintenance mode
+    # MaintainedBackends:
+    #  - "http://s3.dc2.internal"
+    # List request methods to be logged in synclog in case of backend failure
+    SyncLogMethods:
+      - GET
+      - PUT
+      - DELETE
+    # Transports rules with dedicated timeouts
+    Transports:
+      -
+        Name: TransportDef-Method:GET|POST
+        Rules:
+          Method: GET|POST
+          Path: .*
+        Properties:
+          MaxIdleConns: 200
+          MaxIdleConnsPerHost: 1000
+          IdleConnTimeout: 2s
+          ResponseHeaderTimeout: 5s
+      -
+        Name: TransportDef-Method:GET|POST|PUT
+        Rules:
+          Method: GET|POST|PUT
+          QueryParam: acl
+        Properties:
+          MaxIdleConns: 200
+          MaxIdleConnsPerHost: 500
+          IdleConnTimeout: 5s
+          ResponseHeaderTimeout: 5s
+      -
+        Name: OtherTransportDefinition
+        Rules:
+        Properties:
+          MaxIdleConns: 300
+          MaxIdleConnsPerHost: 600
+          IdleConnTimeout: 2s
+          ResponseHeaderTimeout: 2s
 
 # List request methods to be logged in synclog in case of backend failure
 SyncLogMethods:
@@ -212,7 +235,7 @@ It's configured with TechnicalEndpointListen property.
 
 ### Example usage
 
-    curl -vv -X POST -H "Content-Type: application/yaml" --data-binary @akubra.cfg.yaml http://127.0.0.1:8071/validate/configuration
+    curl -vv -X POST -H "Content-Type: application/yaml" --data-binary @akubra.cfg.yaml http://127.0.0.1:8071/configuration/validate
 
 Possible responses:
 
@@ -240,6 +263,20 @@ Response:
     < Content-Length: 2
     OK
 
+## Transports and Rules with dedicated timeouts
+
+This feature guarantees high availability and better transmission.
+
+For example, when one specific HTTP method has lag we can set timeouts with special 'Rule'.
+Another example, when user adds big chunks by multi upload,
+default timeout needs to be changed with dedicated 'Transport' with 'Rule' for this case.
+
+We have 'Rules' for 'Transports' definitions:
+ - required minimum one item in 'Transports' section
+ - required empty or one property (Method, Path, QueryParam) in 'Rules' section
+ - if 'Rules' section is empty, the transport will match any requests
+ - when transport cannot be matched, http 500 error code will be sent to client.
+
 ## Limitations
 
  * User's credentials have to be identical on every backend