Allow common redis and leveldb connections

custom/conf/app.example.ini

@@ -467,8 +467,10 @@ LENGTH = 20
 BATCH_LENGTH = 20
 ; Connection string for redis queues this will store the redis connection string.
 CONN_STR = "addrs=127.0.0.1:6379 db=0"
-; Provide the suffix of the default redis queue name - specific queues can be overriden within in their [queue.name] sections.
+; Provides the suffix of the default redis/disk queue name - specific queues can be overriden within in their [queue.name] sections.
 QUEUE_NAME = "_queue"
+; Provides the suffix of the default redis/disk unique queue set name - specific queues can be overriden within in their [queue.name] sections.
+SET_NAME = "_unique"
 ; If the queue cannot be created at startup - level queues may need a timeout at startup - wrap the queue:
 WRAP_IF_NECESSARY = true
 ; Attempt to create the wrapped queue at max

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -308,15 +308,13 @@ relation to port exhaustion.
 ## Queue (`queue` and `queue.*`)
 
 - `TYPE`: **persistable-channel**: General queue type, currently support: `persistable-channel`, `channel`, `level`, `redis`, `dummy`
-- `DATADIR`: **queues/**: Base DataDir for storing persistent and level queues. `DATADIR` for inidividual queues can be set in `queue.name` sections but will default to `DATADIR/`**`name`**.
+- `DATADIR`: **queues/**: Base DataDir for storing persistent and level queues. `DATADIR` for individual queues can be set in `queue.name` sections but will default to `DATADIR/`**`name`**.
 - `LENGTH`: **20**: Maximal queue size before channel queues block
 - `BATCH_LENGTH`: **20**: Batch data before passing to the handler
-- `CONN_STR`: **addrs=127.0.0.1:6379 db=0**: Connection string for the redis queue type.
-- `QUEUE_NAME`: **_queue**: The suffix for default redis queue name. Individual queues will default to **`name`**`QUEUE_NAME` but can be overriden in the specific `queue.name` section.
-- `SET_NAME`: **_unique**: The suffix that will added to the default redis
-set name for unique queues. Individual queues will default to
-**`name`**`QUEUE_NAME`_`SET_NAME`_ but can be overridden in the specific
-`queue.name` section.
+- `CONN_STR`: **redis://127.0.0.1:6379/0**: Connection string for the redis queue type. Options can be set using query params. Similarly LevelDB options can also be set using: **leveldb://relative/path?option=value** or **leveldb:///absolute/path?option=value**
+- `QUEUE_NAME`: **_queue**: The suffix for default redis and disk queue name. Individual queues will default to **`name`**`QUEUE_NAME` but can be overriden in the specific `queue.name` section.
+- `SET_NAME`: **_unique**: The suffix that will be added to the default redis and disk queue `set` name for unique queues. Individual queues will default to
+ **`name`**`QUEUE_NAME`_`SET_NAME`_ but can be overridden in the specific `queue.name` section.
 - `WRAP_IF_NECESSARY`: **true**: Will wrap queues with a timeoutable queue if the selected queue is not ready to be created - (Only relevant for the level queue.)
 - `MAX_ATTEMPTS`: **10**: Maximum number of attempts to create the wrapped queue
 - `TIMEOUT`: **GRACEFUL_HAMMER_TIME + 30s**: Timeout the creation of the wrapped queue if it takes longer than this to create.
@@ -459,7 +457,7 @@ set name for unique queues. Individual queues will default to
 - `ADAPTER`: **memory**: Cache engine adapter, either `memory`, `redis`, or `memcache`.
 - `INTERVAL`: **60**: Garbage Collection interval (sec), for memory cache only.
 - `HOST`: **\<empty\>**: Connection string for `redis` and `memcache`.
-   - Redis: `network=tcp,addr=127.0.0.1:6379,password=macaron,db=0,pool_size=100,idle_timeout=180`
+   - Redis: `redis://:macaron@127.0.0.1:6379/0?pool_size=100&idle_timeout=180s`
    - Memcache: `127.0.0.1:9090;127.0.0.1:9091`
 - `ITEM_TTL`: **16h**: Time to keep items in cache if not used, Setting it to 0 disables caching.
 
@@ -708,7 +706,7 @@ Task queue configuration has been moved to `queue.task`. However, the below conf
 
 - `QUEUE_TYPE`: **channel**: Task queue type, could be `channel` or `redis`.
 - `QUEUE_LENGTH`: **1000**: Task queue length, available only when `QUEUE_TYPE` is `channel`.
-- `QUEUE_CONN_STR`: **addrs=127.0.0.1:6379 db=0**: Task queue connection string, available only when `QUEUE_TYPE` is `redis`. If redis needs a password, use `addrs=127.0.0.1:6379 password=123 db=0`.
+- `QUEUE_CONN_STR`: **redis://127.0.0.1:6379/0**: Task queue connection string, available only when `QUEUE_TYPE` is `redis`. If redis needs a password, use `redis://123@127.0.0.1:6379/0`.
 
 ## Migrations (`migrations`)
 

go.mod

@@ -38,7 +38,7 @@ require (
 	github.com/go-enry/go-enry/v2 v2.5.2
 	github.com/go-git/go-billy/v5 v5.0.0
 	github.com/go-git/go-git/v5 v5.1.0
-	github.com/go-redis/redis v6.15.2+incompatible
+	github.com/go-redis/redis/v7 v7.4.0
 	github.com/go-sql-driver/mysql v1.5.0
 	github.com/go-swagger/go-swagger v0.25.0
 	github.com/go-testfixtures/testfixtures/v3 v3.4.0
@@ -88,6 +88,7 @@ require (
 	github.com/shurcooL/httpfs v0.0.0-20190527155220-6a4d4a70508b // indirect
 	github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd
 	github.com/stretchr/testify v1.6.1
+	github.com/syndtr/goleveldb v1.0.0
 	github.com/tecbot/gorocksdb v0.0.0-20181010114359-8752a9433481 // indirect
 	github.com/tinylib/msgp v1.1.2 // indirect
 	github.com/tstranex/u2f v1.0.0

go.sum

@@ -342,6 +342,8 @@ github.com/go-openapi/validate v0.19.10 h1:tG3SZ5DC5KF4cyt7nqLVcQXGj5A7mpaYkAcNP
 github.com/go-openapi/validate v0.19.10/go.mod h1:RKEZTUWDkxKQxN2jDT7ZnZi2bhZlbNMAuKvKB+IaGx8=
 github.com/go-redis/redis v6.15.2+incompatible h1:9SpNVG76gr6InJGxoZ6IuuxaCOQwDAhzyXg+Bs+0Sb4=
 github.com/go-redis/redis v6.15.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
+github.com/go-redis/redis/v7 v7.4.0 h1:7obg6wUoj05T0EpY0o8B59S9w5yeMWql7sw2kwNW1x4=
+github.com/go-redis/redis/v7 v7.4.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg=
 github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
 github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
@@ -730,9 +732,13 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
 github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo=
+github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
+github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
@@ -1014,6 +1020,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=

modules/cache/cache.go

@@ -13,7 +13,6 @@ import (
 	mc "gitea.com/macaron/cache"
 
 	_ "gitea.com/macaron/cache/memcache" // memcache plugin for cache
-	_ "gitea.com/macaron/cache/redis"
 )
 
 var (

vendor/gitea.com/macaron/cache/redis/redis.go → modules/cache/cache_redis.go

@@ -1,35 +1,23 @@
-// Copyright 2013 Beego Authors
-// Copyright 2014 The Macaron Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License"): you may
-// not use this file except in compliance with the License. You may obtain
-// a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-// License for the specific language governing permissions and limitations
-// under the License.
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
 
 package cache
 
 import (
 	"fmt"
-	"strings"
 	"time"
 
-	"github.com/go-redis/redis"
-	"github.com/unknwon/com"
-	"gopkg.in/ini.v1"
+	"code.gitea.io/gitea/modules/nosql"
 
 	"gitea.com/macaron/cache"
+	"github.com/go-redis/redis/v7"
+	"github.com/unknwon/com"
 )
 
 // RedisCacher represents a redis cache adapter implementation.
 type RedisCacher struct {
-	c          *redis.Client
+	c          redis.UniversalClient
 	prefix     string
 	hsetName   string
 	occupyMode bool
@@ -112,7 +100,7 @@ func (c *RedisCacher) IsExist(key string) bool {
 // Flush deletes all cached data.
 func (c *RedisCacher) Flush() error {
 	if c.occupyMode {
-		return c.c.FlushDb().Err()
+		return c.c.FlushDB().Err()
 	}
 
 	keys, err := c.c.HKeys(c.hsetName).Result()
@@ -131,46 +119,20 @@ func (c *RedisCacher) StartAndGC(opts cache.Options) error {
 	c.hsetName = "MacaronCache"
 	c.occupyMode = opts.OccupyMode
 
-	cfg, err := ini.Load([]byte(strings.Replace(opts.AdapterConfig, ",", "\n", -1)))
-	if err != nil {
-		return err
-	}
+	uri := nosql.ToRedisURI(opts.AdapterConfig)
 
-	opt := &redis.Options{
-		Network: "tcp",
-	}
-	for k, v := range cfg.Section("").KeysHash() {
+	c.c = nosql.GetManager().GetRedisClient(uri.String())
+
+	for k, v := range uri.Query() {
 		switch k {
-		case "network":
-			opt.Network = v
-		case "addr":
-			opt.Addr = v
-		case "password":
-			opt.Password = v
-		case "db":
-			opt.DB = com.StrTo(v).MustInt()
-		case "pool_size":
-			opt.PoolSize = com.StrTo(v).MustInt()
-		case "idle_timeout":
-			opt.IdleTimeout, err = time.ParseDuration(v + "s")
-			if err != nil {
-				return fmt.Errorf("error parsing idle timeout: %v", err)
-			}
 		case "hset_name":
-			c.hsetName = v
+			c.hsetName = v[0]
 		case "prefix":
-			c.prefix = v
-		default:
-			return fmt.Errorf("session/redis: unsupported option '%s'", k)
+			c.prefix = v[0]
 		}
 	}
 
-	c.c = redis.NewClient(opt)
-	if err = c.c.Ping().Err(); err != nil {
-		return err
-	}
-
-	return nil
+	return c.c.Ping().Err()
 }
 
 func init() {

modules/nosql/leveldb.go

@@ -0,0 +1,25 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package nosql
+
+import "net/url"
+
+// ToLevelDBURI converts old style connections to a LevelDBURI
+//
+// A LevelDBURI matches the pattern:
+//
+// leveldb://path[?[option=value]*]
+//
+// We have previously just provided the path but this prevent other options
+func ToLevelDBURI(connection string) *url.URL {
+	uri, err := url.Parse(connection)
+	if err == nil && uri.Scheme == "leveldb" {
+		return uri
+	}
+	uri, _ = url.Parse("leveldb://common")
+	uri.Host = ""
+	uri.Path = connection
+	return uri
+}

modules/nosql/manager.go

@@ -0,0 +1,71 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package nosql
+
+import (
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/go-redis/redis/v7"
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+var manager *Manager
+
+// Manager is the nosql connection manager
+type Manager struct {
+	mutex sync.Mutex
+
+	RedisConnections   map[string]*redisClientHolder
+	LevelDBConnections map[string]*levelDBHolder
+}
+
+type redisClientHolder struct {
+	redis.UniversalClient
+	name  []string
+	count int64
+}
+
+func (r *redisClientHolder) Close() error {
+	return manager.CloseRedisClient(r.name[0])
+}
+
+type levelDBHolder struct {
+	name  []string
+	count int64
+	db    *leveldb.DB
+}
+
+func init() {
+	_ = GetManager()
+}
+
+// GetManager returns a Manager and initializes one as singleton is there's none yet
+func GetManager() *Manager {
+	if manager == nil {
+		manager = &Manager{
+			RedisConnections:   make(map[string]*redisClientHolder),
+			LevelDBConnections: make(map[string]*levelDBHolder),
+		}
+	}
+	return manager
+}
+
+func valToTimeDuration(vs []string) (result time.Duration) {
+	var err error
+	for _, v := range vs {
+		result, err = time.ParseDuration(v)
+		if err != nil {
+			var val int
+			val, err = strconv.Atoi(v)
+			result = time.Duration(val)
+		}
+		if err == nil {
+			return
+		}
+	}
+	return
+}