diff --git a/README.md b/README.md index 5cd0195f09..3c97ae41bd 100644 --- a/README.md +++ b/README.md @@ -420,7 +420,7 @@ re-forwarded to the server. However if it's not restored, then the server connection is considered dead, and all outstanding requests are re-scheduled to other servers and/or connections. -If a server connection fails intermittenly, then requests may sit in the +If a server connection fails intermittently, then requests may sit in the connection's forwarding queue for some time. The following directives set certain allowed limits before these requests are considered failed: ``` @@ -481,7 +481,7 @@ called `default`. All server-related directives listed in [Servers](#Servers) section above are applicable for definition for a server group. Also, a scheduler may be -speficied for a group. +specified for a group. Below is an example of server group definition: ``` @@ -618,11 +618,41 @@ then the default match rule is added to route HTTP requests to the group dropped. +#### Sticky Sessions + +In addition to schedulers Tempesta can also use [Sticky Cookies](sticky-cookie) +for load distribution. In this method unique clients are pinned to the +upstream servers. Method can't be applied if client doesn't support cookies. +Not used in default configuration. + +Client's first request to a server group will be forwarded to a server chosen +by the group scheduler. All the following requests to the server group will +be forwarded to the same server. If server goes down (for a maintenance or +due to networking errors) client receives `502` responses. When the server +is back online it will continue serving this client. + +Session persistence is the highest priority for the method. So if the whole +primary server group is offline new sessions will be pinned to a server in the +backup group if applied. Backup server will continue serving the client even +when the primary group is back online. That means that switching from backup +server group back to the primary group ends only after all the current +sessions pinned to backup server group are expired. + +`allow_failover` option allow Tempesta pin sessions to a new server if +the current pinned server went offline. Accident will be logged. Moving client +session from one server to another actually brakes session persistence, so +the backend application must support the feature. + +Note, that method does not support setting different backup server groups for +the same primary groups in [HTTP Scheduler](http-scheduler). + + ### Sticky Cookie **Sticky cookie** is a special HTTP cookie that is generated by Tempesta. It allows for unique identification of each client or can be used as challenge cookie for simple L7 DDoS mitigation when bots are unable to process cookies. +It is also used for a [load balancing](sticky-sessions). When used, Tempesta sticky cookie is expected in HTTP requests. Otherwise, Tempesta asks in an HTTP response that sticky cookie is present in diff --git a/etc/tempesta_fw.conf b/etc/tempesta_fw.conf index 168f9bb58a..f469d6a5c7 100644 --- a/etc/tempesta_fw.conf +++ b/etc/tempesta_fw.conf @@ -13,10 +13,10 @@ # SCHED_NAME is a name of a scheduler module that distributes the load # among servers within a group. There are two schedulers available: # - "round-robin" (default) - rotates all servers in the group in -# the round-robin manner, so requests are distributed uniformely across +# the round-robin manner, so requests are distributed uniformly across # servers. # - "hash" - chooses a server based on a URI/Host hash of a request. -# Requests are still distributed uniformely, but a request with the same +# Requests are still distributed uniformly, but a request with the same # URI/Host is always sent to the same server. # # Note that there's also the HTTP scheduler. It dispatches requests among @@ -140,7 +140,7 @@ # Syntax: # server_queue_size 600; # -# This is the maxumum number of requests that may be in the forwarding +# This is the maximum number of requests that may be in the forwarding # queue of a server connection at any given time. # # Default: @@ -494,6 +494,39 @@ # Example: # sess_lifetime 900; +# TAG: sticky_sessions +# +# Load balancing method. Forward all requests from client to the the same +# upstream server. Applied to a server group. +# +# Syntax: +# sticky_sessions [allow_failover] +# +# Default: +# Sticky sessions are disabled; +# +# First request to a server group in client's HTTP session will be forwarded +# to upstream server chosen by scheduler applied to the server group. All the +# following requests to that group in this session will pass through scheduler +# and will be forwarded to the same server. Client will recieve 502 error +# if pinned upstream server went offline. Method can't be applied if client +# does not support cookies or Tempesta sticky cookie is disabled. +# +# Note: Session preservation is the priority for the method, so client will +# be served by backup server group even when primary group is back online. +# +# allow_failover - Pin session to a new upstream server and print the warning +# if the last pinned server went offline instead of sending 502 error to client. +# +# Examples: +# srv_group app { +# server 10.10.0.1:8080; +# server 10.10.0.2:8080; +# server [fc00::3]:8081 conns_n=1; +# +# sticky_sessions; +# } + # # Frang configuration. # diff --git a/tempesta_fw/http.c b/tempesta_fw/http.c index fdbafc0107..9435e5357d 100644 --- a/tempesta_fw/http.c +++ b/tempesta_fw/http.c @@ -25,6 +25,7 @@ #include "client.h" #include "hash.h" #include "http_msg.h" +#include "http_sess.h" #include "log.h" #include "procfs.h" #include "server.h" @@ -1599,6 +1600,9 @@ tfw_http_adjust_resp(TfwHttpResp *resp, TfwHttpReq *req) return r; r = tfw_http_add_hdr_via(hm); + if (r < 0) + return r; + if (resp->flags & TFW_HTTP_RESP_STALE) { #define S_WARN_110 "Warning: 110 - Response is stale" /* TODO: ajust for #215 */ @@ -1819,14 +1823,6 @@ tfw_http_req_cache_cb(TfwHttpReq *req, TfwHttpResp *resp) * executed, to avoid unnecessary work in SoftIRQ and to speed up * the cache operation. At the same time, cache hits are expected * to prevail over cache misses, so this is not a frequent path. - * - * TODO #593: check whether req->sess->srv_conn is alive. If not, - * then get a new connection for req->sess->srv_conn->peer from - * an appropriate scheduler. That eliminates the long generic - * scheduling work flow. When the first request in a session is - * scheduled by the generic logic, TfwSession->srv_conn must be - * initialized to point at the appropriate TfwConn{}, so that - * all subsequent session hits are scheduled much faster. */ if (!(srv_conn = tfw_sched_get_srv_conn((TfwMsg *)req))) { TFW_WARN("Unable to find a back end server\n"); diff --git a/tempesta_fw/http.h b/tempesta_fw/http.h index ed6d47cf52..70de05650e 100644 --- a/tempesta_fw/http.h +++ b/tempesta_fw/http.h @@ -26,9 +26,12 @@ #include "connection.h" #include "gfsm.h" #include "msg.h" +#include "server.h" #include "str.h" #include "vhost.h" +typedef struct tfw_http_sess_t TfwHttpSess; + /** * HTTP Generic FSM states. * @@ -269,25 +272,6 @@ typedef struct { /* It is stale, but pass with a warning */ #define TFW_HTTP_RESP_STALE 0x040000 -/** - * HTTP session descriptor. - * - * @hmac - crypto hash from values of an HTTP request; - * @hentry - hash list entry for all sessions hash; - * @users - the session use counter; - * @ts - timestamp for the client's session; - * @expire - expiration time for the session; - * @srv_conn - upstream server connection servicing the session; - */ -typedef struct { - unsigned char hmac[SHA1_DIGEST_SIZE]; - struct hlist_node hentry; - atomic_t users; - unsigned long ts; - unsigned long expires; - TfwSrvConn *srv_conn; -} TfwHttpSess; - /* * The structure to hold data for an HTTP error response. * An error response is sent later in an unlocked queue context. @@ -488,11 +472,4 @@ int tfw_http_send_504(TfwHttpReq *req, const char *reason); void *tfw_msg_setup(TfwHttpMsg *hm, size_t len); void tfw_msg_add_data(void *handle, TfwMsg *msg, char *data, size_t len); -/* - * HTTP session routines. - */ -int tfw_http_sess_obtain(TfwHttpReq *req); -int tfw_http_sess_resp_process(TfwHttpResp *resp, TfwHttpReq *req); -void tfw_http_sess_put(TfwHttpSess *sess); - #endif /* __TFW_HTTP_H__ */ diff --git a/tempesta_fw/http_sess.c b/tempesta_fw/http_sess.c index 6f82a435c5..d5cdf5a0c3 100644 --- a/tempesta_fw/http_sess.c +++ b/tempesta_fw/http_sess.c @@ -44,6 +44,7 @@ #include "client.h" #include "hash.h" #include "http_msg.h" +#include "http_sess.h" #define STICKY_NAME_MAXLEN (32) #define STICKY_NAME_DEFAULT "__tfw" @@ -55,7 +56,7 @@ /** * @name - name of sticky cookie; * @name_eq - @name plus "=" to make some operations faster; - * @sess_lifetime - sesscion lifetime in seconds; + * @sess_lifetime - session lifetime in seconds; */ typedef struct { TfwStr name; @@ -83,7 +84,7 @@ static TfwCfgSticky tfw_cfg_sticky; static struct crypto_shash *tfw_sticky_shash; static char tfw_sticky_key[STICKY_KEY_MAXLEN]; -SessHashBucket sess_hash[SESS_HASH_SZ] = { +static SessHashBucket sess_hash[SESS_HASH_SZ] = { [0 ... (SESS_HASH_SZ - 1)] = { HLIST_HEAD_INIT, } @@ -137,7 +138,7 @@ static int search_cookie(TfwPool *pool, const TfwStr *cookie, TfwStr *val) { const char *const cstr = tfw_cfg_sticky.name_eq.ptr; - const unsigned int clen = tfw_cfg_sticky.name_eq.len; + const unsigned long clen = tfw_cfg_sticky.name_eq.len; TfwStr *chunk, *end, *next; TfwStr tmp = { .flags = 0, }; unsigned int n = TFW_STR_CHUNKN(cookie); @@ -310,7 +311,7 @@ static int tfw_http_sticky_add(TfwHttpResp *resp, TfwHttpReq *req) { static const unsigned int len = sizeof(StickyVal) * 2; - unsigned int r; + int r; TfwHttpSess *sess = req->sess; unsigned long ts_be64 = cpu_to_be64(sess->ts); char buf[len]; @@ -581,7 +582,10 @@ tfw_http_sess_obtain(TfwHttpReq *req) sess->expires = tfw_cfg_sticky.sess_lifetime ? sv.ts + tfw_cfg_sticky.sess_lifetime * HZ : 0; - sess->srv_conn = NULL; /* TODO #593 not scheduled yet */ + sess->st_conn.srv_conn = NULL; + sess->st_conn.main_sg = NULL; + sess->st_conn.backup_sg = NULL; + rwlock_init(&sess->st_conn.lock); TFW_DBG("new session %p\n", sess); @@ -595,39 +599,132 @@ tfw_http_sess_obtain(TfwHttpReq *req) return 0; } +/** + * Try to reuse last used connection or last used server. + */ +static inline TfwSrvConn * +__try_conn(TfwMsg *msg, TfwStickyConn *st_conn) +{ + TfwServer *srv; + + if (unlikely(!st_conn->srv_conn)) + return NULL; + + if (!tfw_srv_conn_restricted(st_conn->srv_conn) + && !tfw_srv_conn_queue_full(st_conn->srv_conn) + && !tfw_srv_conn_hasnip(st_conn->srv_conn) + && tfw_srv_conn_get_if_live(st_conn->srv_conn)) + { + return st_conn->srv_conn; + } + + /* Try to sched from the same server. */ + srv = (TfwServer *)st_conn->srv_conn->peer; + + return srv->sg->sched->sched_srv_conn(msg, srv); +} + +/** + * Find an outgoing connection for client with tempesta sticky cookie. + * @sess is not null when calling the function. + * + * Reuse req->sess->st_conn.srv_conn if it is alive. If not, + * then get a new connection for req->sess->srv_conn->peer from + * an appropriate scheduler. That eliminates the long generic + * scheduling work flow. + */ +TfwSrvConn * +tfw_http_sess_get_srv_conn(TfwMsg *msg) +{ + TfwHttpSess *sess = ((TfwHttpReq *)msg)->sess; + TfwStickyConn *st_conn; + TfwSrvConn *srv_conn; + + BUG_ON(!sess); + st_conn = &sess->st_conn; + + read_lock(&st_conn->lock); + + if ((srv_conn = __try_conn(msg, st_conn))) { + read_unlock(&st_conn->lock); + return srv_conn; + } + + read_unlock(&st_conn->lock); + + if (st_conn->srv_conn) { + /* Failed to sched from the same server. */ + TfwServer *srv = (TfwServer *)st_conn->srv_conn->peer; + char addr_str[TFW_ADDR_STR_BUF_SIZE] = { 0 }; + + tfw_addr_ntop(&srv->addr, addr_str, sizeof(addr_str)); + + if (!(srv->sg->flags & TFW_SRV_STICKY_FAILOVER)) { + TFW_ERR("sched %s: Unable to schedule new request in " + "session to server %s in group %s\n", + srv->sg->sched->name, addr_str, srv->sg->name); + return NULL; + } + else { + TFW_WARN("sched %s: Unable to schedule new request in " + "session to server %s in group %s," + " fallback to a new server\n", + srv->sg->sched->name, addr_str, srv->sg->name); + } + } + + write_lock(&st_conn->lock); + /* + * Connection and server may return back online while we were trying + * for a lock. + */ + if ((srv_conn = __try_conn(msg, st_conn))) + goto done; + + if (st_conn->main_sg) + srv_conn = tfw_sched_get_sg_srv_conn(msg, st_conn->main_sg, + st_conn->backup_sg); + else + srv_conn = __tfw_sched_get_srv_conn(msg); + + /* + * Save connection into session only if used server group is configured + * to have sticky connections. + */ + if (srv_conn + && (((TfwServer *)srv_conn->peer)->sg->flags & TFW_SRV_STICKY)) { + st_conn->srv_conn = srv_conn; + } + +done: + write_unlock(&st_conn->lock); + + return srv_conn; +} + int __init tfw_http_sess_init(void) { - int ret, i; + int i, ret = -ENOMEM; u_char *ptr; - if ((ptr = kzalloc(STICKY_NAME_MAXLEN + 1, GFP_KERNEL)) == NULL) { + if ((ptr = kzalloc(STICKY_NAME_MAXLEN + 1, GFP_KERNEL)) == NULL) return -ENOMEM; - } + tfw_cfg_sticky.name.ptr = tfw_cfg_sticky.name_eq.ptr = ptr; tfw_cfg_sticky.name.len = tfw_cfg_sticky.name_eq.len = 0; tfw_sticky_shash = crypto_alloc_shash("hmac(sha1)", 0, 0); if (IS_ERR(tfw_sticky_shash)) { pr_err("shash allocation failed\n"); - return PTR_ERR(tfw_sticky_shash); - } - - get_random_bytes(tfw_sticky_key, sizeof(tfw_sticky_key)); - ret = crypto_shash_setkey(tfw_sticky_shash, - (u8 *)tfw_sticky_key, - sizeof(tfw_sticky_key)); - if (ret) { - crypto_free_shash(tfw_sticky_shash); - return ret; + ret = (int)PTR_ERR(tfw_sticky_shash); + goto err; } sess_cache = kmem_cache_create("tfw_sess_cache", sizeof(TfwHttpSess), 0, 0, NULL); - if (!sess_cache) { - crypto_free_shash(tfw_sticky_shash); - return -ENOMEM; - } + if (!sess_cache) + goto err_shash; /* * Dynamically initialize hash table spinlocks to avoid lockdep leakage @@ -637,6 +734,12 @@ tfw_http_sess_init(void) spin_lock_init(&sess_hash[i].lock); return 0; + +err_shash: + crypto_free_shash(tfw_sticky_shash); +err: + kfree(tfw_cfg_sticky.name.ptr); + return ret; } void @@ -703,15 +806,23 @@ tfw_http_sticky_cfg(TfwCfgSpec *cs, TfwCfgEntry *ce) static int tfw_http_sticky_secret_cfg(TfwCfgSpec *cs, TfwCfgEntry *ce) { - int r, len = strlen(ce->vals[0]); + int r; + unsigned int len = (unsigned int)strlen(ce->vals[0]); if (tfw_cfg_check_single_val(ce)) return -EINVAL; if (len > STICKY_KEY_MAXLEN) return -EINVAL; - memset(tfw_sticky_key, 0, STICKY_KEY_MAXLEN); - memcpy(tfw_sticky_key, ce->vals[0], len); + if (len) { + memset(tfw_sticky_key, 0, STICKY_KEY_MAXLEN); + memcpy(tfw_sticky_key, ce->vals[0], len); + } + else { + get_random_bytes(tfw_sticky_key, sizeof(tfw_sticky_key)); + len = sizeof(tfw_sticky_key); + } + r = crypto_shash_setkey(tfw_sticky_shash, (u8 *)tfw_sticky_key, len); if (r) { crypto_free_shash(tfw_sticky_shash); @@ -720,6 +831,23 @@ tfw_http_sticky_secret_cfg(TfwCfgSpec *cs, TfwCfgEntry *ce) return 0; } +static int +tfw_http_sticky_sess_lifetime_cfg(TfwCfgSpec *cs, TfwCfgEntry *ce) +{ + int r; + cs->dest = &tfw_cfg_sticky.sess_lifetime; + + r = tfw_cfg_set_int(cs, ce); + /* + * "sess_lifetime 0;" means unlimited session lifetime, + * set tfw_cfg_sticky.sess_lifetime to maximum value. + */ + if (!r && !tfw_cfg_sticky.sess_lifetime) + tfw_cfg_sticky.sess_lifetime = UINT_MAX; + + return r; +} + TfwCfgMod tfw_http_sess_cfg_mod = { .name = "http_sticky", .start = tfw_cfg_sess_start, @@ -732,16 +860,20 @@ TfwCfgMod tfw_http_sess_cfg_mod = { }, { .name = "sticky_secret", + .deflt = "\"\"", .handler = tfw_http_sticky_secret_cfg, .allow_none = true, }, { + /* Value is parsed as int, set max to INT_MAX*/ .name = "sess_lifetime", .deflt = "0", - .handler = tfw_cfg_set_int, - .dest = &tfw_cfg_sticky.sess_lifetime, + .handler = tfw_http_sticky_sess_lifetime_cfg, + .spec_ext = &(TfwCfgSpecInt) { + .range = { 0, INT_MAX }, + }, .allow_none = true, }, - {} + { 0 } } }; diff --git a/tempesta_fw/http_sess.h b/tempesta_fw/http_sess.h new file mode 100644 index 0000000000..3252c2ab6e --- /dev/null +++ b/tempesta_fw/http_sess.h @@ -0,0 +1,82 @@ +/* + * Tempesta FW + * + * Copyright (C) 2017 Tempesta Technologies, Inc. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, + * or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 59 + * Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +#ifndef __TFW_HTTP_SESS_H__ +#define __TFW_HTTP_SESS_H__ + +#include "http.h" + +/** + * Connection, servicing HTTP session. + * + * @srv_conn - last used connection; + * @main_sg - primary server group to schedule connection on failovering; + * @backup_sg - backup server group; + * @lock - protects whole @TfwStickyConn; + */ +typedef struct { + TfwSrvConn *srv_conn; + TfwSrvGroup *main_sg; + TfwSrvGroup *backup_sg; + rwlock_t lock; +} TfwStickyConn; + +/** + * HTTP session descriptor. + * + * @hmac - crypto hash from values of an HTTP request; + * @hentry - hash list entry for all sessions hash; + * @users - the session use counter; + * @ts - timestamp for the client's session; + * @expire - expiration time for the session; + * @st_conn - upstream server connection servicing the session; + */ +struct tfw_http_sess_t { + unsigned char hmac[SHA1_DIGEST_SIZE]; + struct hlist_node hentry; + atomic_t users; + unsigned long ts; + unsigned long expires; + TfwStickyConn st_conn; +}; + +int tfw_http_sess_obtain(TfwHttpReq *req); +int tfw_http_sess_resp_process(TfwHttpResp *resp, TfwHttpReq *req); +void tfw_http_sess_put(TfwHttpSess *sess); + +/* Sticky sessions scheduling routines. */ +TfwSrvConn *tfw_http_sess_get_srv_conn(TfwMsg *msg); + +static inline void +tfw_http_sess_save_sg(TfwHttpReq *req, TfwSrvGroup *main_sg, + TfwSrvGroup *backup_sg) +{ + if (req->sess && (main_sg->flags & TFW_SRV_STICKY)) { + TfwStickyConn *st_conn = &req->sess->st_conn; + + /* + * @st_conn->lock is already acquired for writing, if called + * during @tfw_http_sess_get_srv_conn() routine. + */ + st_conn->main_sg = main_sg; + st_conn->backup_sg = backup_sg; + } +} + +#endif /* __TFW_HTTP_SESS_H__ */ diff --git a/tempesta_fw/sched.c b/tempesta_fw/sched.c index df13b33e11..2ef91b4ddf 100644 --- a/tempesta_fw/sched.c +++ b/tempesta_fw/sched.c @@ -23,6 +23,7 @@ #include "tempesta_fw.h" #include "log.h" #include "server.h" +#include "http_sess.h" /* * Normally, schedulers are separate modules. Schedulers register @@ -39,7 +40,41 @@ static LIST_HEAD(sched_list); static DEFINE_SPINLOCK(sched_lock); -/* +/** + * Find connection for a message @msg in @main_sg or @backup_sg server groups. + * + * If client support cookies the function is called under session's write lock, + * and updates to session informations are possible. + */ +TfwSrvConn * +tfw_sched_get_sg_srv_conn(TfwMsg *msg, TfwSrvGroup *main_sg, + TfwSrvGroup *backup_sg) +{ + TfwHttpReq *req = (TfwHttpReq *)msg; + TfwSrvConn *srv_conn; + + BUG_ON(!main_sg); + TFW_DBG2("sched: use server group: '%s'\n", sg->name); + + tfw_http_sess_save_sg(req, main_sg, backup_sg); + + srv_conn = main_sg->sched->sched_sg_conn(msg, main_sg); + + if (unlikely(!srv_conn && backup_sg)) { + TFW_DBG("sched: the main group is offline, use backup: '%s'\n", + sg->name); + srv_conn = backup_sg->sched->sched_sg_conn(msg, backup_sg); + } + + if (unlikely(!srv_conn)) + TFW_DBG2("sched: Unable to select server from group '%s'\n", + sg->name); + + return srv_conn; +} +EXPORT_SYMBOL(tfw_sched_get_sg_srv_conn); + +/** * Find an outgoing connection for an HTTP message. * * Where an HTTP message goes in controlled by schedulers. It may @@ -48,11 +83,9 @@ static DEFINE_SPINLOCK(sched_lock); * is received. Schedulers that distribute HTTP messages among * server groups come first in the list. The search stops when * these schedulers run out. - * - * This function is always called in SoftIRQ context. */ TfwSrvConn * -tfw_sched_get_srv_conn(TfwMsg *msg) +__tfw_sched_get_srv_conn(TfwMsg *msg) { TfwSrvConn *srv_conn; TfwScheduler *sched; @@ -71,6 +104,24 @@ tfw_sched_get_srv_conn(TfwMsg *msg) return NULL; } +/* + * Find an outgoing server connection for an HTTP message. + * + * This function is always called in SoftIRQ context. + */ +TfwSrvConn * +tfw_sched_get_srv_conn(TfwMsg *msg) +{ + TfwHttpReq *req = (TfwHttpReq *)msg; + TfwHttpSess *sess = req->sess; + + /* Sticky cookies are disabled or client doesn't support cookies. */ + if (!sess) + return __tfw_sched_get_srv_conn(msg); + + return tfw_http_sess_get_srv_conn(msg); +} + /* * Lookup a scheduler by name. * @@ -132,5 +183,7 @@ tfw_sched_unregister(TfwScheduler *sched) /* Make sure the removed @sched is not used. */ synchronize_rcu(); + /* Clear up scheduler for future use. */ + INIT_LIST_HEAD(&sched->list); } EXPORT_SYMBOL(tfw_sched_unregister); diff --git a/tempesta_fw/sched/tfw_sched_hash.c b/tempesta_fw/sched/tfw_sched_hash.c index c5d110f48a..7fd6e90b2c 100644 --- a/tempesta_fw/sched/tfw_sched_hash.c +++ b/tempesta_fw/sched/tfw_sched_hash.c @@ -45,22 +45,25 @@ MODULE_AUTHOR(TFW_AUTHOR); MODULE_DESCRIPTION("Tempesta hash-based scheduler"); -MODULE_VERSION("0.2.1"); +MODULE_VERSION("0.3.0"); MODULE_LICENSE("GPL"); typedef struct { - TfwSrvConn *srv_conn; - unsigned long hash; -} TfwConnHash; + size_t conn_n; + TfwServer *srv; + TfwSrvConn *conn[TFW_SRV_MAX_CONN]; + unsigned long hash[TFW_SRV_MAX_CONN]; +} TfwHashSrv; -/* The last item is used as the list teminator. */ -#define __HLIST_SZ(n) ((n) + 1) -#define __HDATA_SZ(n) (__HLIST_SZ(n) * sizeof(TfwConnHash)) +typedef struct { + size_t srv_n; + TfwHashSrv srvs[TFW_SG_MAX_SRV]; +} TfwHashSrvList; static void tfw_sched_hash_alloc_data(TfwSrvGroup *sg) { - sg->sched_data = kzalloc(__HDATA_SZ(TFW_SG_MAX_CONN), GFP_KERNEL); + sg->sched_data = kzalloc(sizeof(TfwHashSrvList), GFP_KERNEL); BUG_ON(!sg->sched_data); } @@ -99,20 +102,65 @@ __calc_conn_hash(TfwServer *srv, size_t conn_idx) } static void -tfw_sched_hash_add_conn(TfwSrvGroup *sg, TfwServer *srv, TfwSrvConn *srv_conn) +tfw_sched_hash_add_conn(TfwSrvGroup *sg, TfwServer *srv, TfwSrvConn *conn) +{ + size_t s, c; + TfwHashSrv *srv_cl; + TfwHashSrvList *sl = sg->sched_data; + + BUG_ON(!sl); + + for (s = 0; s < sl->srv_n; ++s) + if (sl->srvs[s].srv == srv) + break; + if (s == sl->srv_n) { + sl->srvs[s].srv = srv; + ++sl->srv_n; + BUG_ON(sl->srv_n > TFW_SG_MAX_SRV); + srv->sched_data = &sl->srvs[s]; + } + + srv_cl = &sl->srvs[s]; + + for (c = 0; c < srv_cl->conn_n; ++c) + if (srv_cl->conn[c] == conn) { + TFW_WARN("sched_hash: Try to add existing connection," + " srv=%zu conn=%zu\n", s, c); + return; + } + srv_cl->conn[c] = conn; + srv_cl->hash[c] = __calc_conn_hash(srv, s * TFW_SRV_MAX_CONN + c); + ++srv_cl->conn_n; + BUG_ON(srv_cl->conn_n > TFW_SRV_MAX_CONN); +} + +static inline void +__find_best_conn(TfwSrvConn **best_conn, TfwHashSrv *srv_cl, + unsigned long *best_weight, unsigned long msg_hash) { size_t i; - TfwConnHash *conn_hash = sg->sched_data; - BUG_ON(!conn_hash); - for (i = 0; i < __HLIST_SZ(TFW_SG_MAX_CONN); ++i) { - if (conn_hash[i].srv_conn) + for (i = 0; i < srv_cl->conn_n; ++i) { + unsigned long curr_weight; + TfwSrvConn *conn = srv_cl->conn[i]; + + if (unlikely(tfw_srv_conn_restricted(conn) + || tfw_srv_conn_queue_full(conn) + || !tfw_srv_conn_live(conn))) continue; - conn_hash[i].srv_conn = srv_conn; - conn_hash[i].hash = __calc_conn_hash(srv, i); - return; + + curr_weight = msg_hash ^ srv_cl->hash[i]; + /* + * XOR might return 0, more or equal comparisson is required. + * If server have only one active connection it still may + * be the best connecton to serve the request. More likely + * to happen when serving via @tfw_sched_hash_get_srv_conn(). + */ + if (curr_weight >= *best_weight) { + *best_weight = curr_weight; + *best_conn = conn; + } } - BUG(); } /** @@ -136,33 +184,66 @@ tfw_sched_hash_add_conn(TfwSrvGroup *sg, TfwServer *srv, TfwSrvConn *srv_conn) * a matching one with the highest weight. That adds some overhead. */ static TfwSrvConn * -tfw_sched_hash_get_srv_conn(TfwMsg *msg, TfwSrvGroup *sg) +tfw_sched_hash_get_sg_conn(TfwMsg *msg, TfwSrvGroup *sg) { - unsigned long tries, msg_hash, curr_weight, best_weight = 0; - TfwSrvConn *best_srv_conn = NULL; - TfwConnHash *ch; + unsigned long msg_hash; + unsigned long tries = TFW_SG_MAX_CONN; + TfwHashSrvList *sl = sg->sched_data; + + BUG_ON(!sl); msg_hash = tfw_http_req_key_calc((TfwHttpReq *)msg); - for (tries = 0; tries < __HLIST_SZ(TFW_SG_MAX_CONN); ++tries) { - for (ch = sg->sched_data; ch->srv_conn; ++ch) { - if (unlikely(tfw_srv_conn_restricted(ch->srv_conn) - || tfw_srv_conn_queue_full(ch->srv_conn) - || !tfw_srv_conn_live(ch->srv_conn))) - continue; - curr_weight = msg_hash ^ ch->hash; - if (curr_weight > best_weight) { - best_weight = curr_weight; - best_srv_conn = ch->srv_conn; - } + while (--tries) { + size_t i; + unsigned long best_weight = 0; + TfwSrvConn *best_conn = NULL; + + for (i = 0; i < sl->srv_n; ++i) { + TfwHashSrv *srv_cl = &sl->srvs[i]; + __find_best_conn(&best_conn, srv_cl, &best_weight, + msg_hash); } - if (unlikely(!best_srv_conn)) + if (unlikely(!best_conn)) return NULL; - if (likely(tfw_srv_conn_get_if_live(best_srv_conn))) - return best_srv_conn; + if (likely(tfw_srv_conn_get_if_live(best_conn))) + return best_conn; } return NULL; } +/** + * Same as @tfw_sched_hash_get_sg_conn(), but schedule for a specific server + * in a group. + */ +static TfwSrvConn * +tfw_sched_hash_get_srv_conn(TfwMsg *msg, TfwServer *srv) +{ + unsigned long msg_hash; + size_t tries; + TfwHashSrv *srv_cl = srv->sched_data; + + /* + * For @srv without connections srv_cl will be NULL, that normally + * does not happen in real life, but unit tests check that case. + */ + if (unlikely(!srv_cl)) + return NULL; + + msg_hash = tfw_http_req_key_calc((TfwHttpReq *)msg); + /* Try several times even if server has just a few connections. */ + tries = srv_cl->conn_n + 1; + while (--tries) { + unsigned long best_weight = 0; + TfwSrvConn *best_conn = NULL; + + __find_best_conn(&best_conn, srv_cl, &best_weight, msg_hash); + if (unlikely(!best_conn)) + return NULL; + if (likely(tfw_srv_conn_get_if_live(best_conn))) + return best_conn; + } + return NULL; +} static TfwScheduler tfw_sched_hash = { .name = "hash", @@ -170,7 +251,8 @@ static TfwScheduler tfw_sched_hash = { .add_grp = tfw_sched_hash_alloc_data, .del_grp = tfw_sched_hash_free_data, .add_conn = tfw_sched_hash_add_conn, - .sched_srv = tfw_sched_hash_get_srv_conn, + .sched_sg_conn = tfw_sched_hash_get_sg_conn, + .sched_srv_conn = tfw_sched_hash_get_srv_conn, }; int diff --git a/tempesta_fw/sched/tfw_sched_http.c b/tempesta_fw/sched/tfw_sched_http.c index c6b9648954..76adb3ce10 100644 --- a/tempesta_fw/sched/tfw_sched_http.c +++ b/tempesta_fw/sched/tfw_sched_http.c @@ -82,7 +82,7 @@ MODULE_AUTHOR(TFW_AUTHOR); MODULE_DESCRIPTION("Tempesta HTTP scheduler"); -MODULE_VERSION("0.2.1"); +MODULE_VERSION("0.3.0"); MODULE_LICENSE("GPL"); typedef struct { @@ -102,8 +102,6 @@ static TfwHttpMatchList *tfw_sched_http_rules; static TfwSrvConn * tfw_sched_http_sched_grp(TfwMsg *msg) { - TfwSrvGroup *sg; - TfwSrvConn *srv_conn; TfwSchedHttpRule *rule; if(!tfw_sched_http_rules || list_empty(&tfw_sched_http_rules->list)) @@ -116,38 +114,30 @@ tfw_sched_http_sched_grp(TfwMsg *msg) return NULL; } - sg = rule->main_sg; - BUG_ON(!sg); - TFW_DBG2("sched_http: use server group: '%s'\n", sg->name); - - srv_conn = sg->sched->sched_srv(msg, sg); - - if (unlikely(!srv_conn && rule->backup_sg)) { - sg = rule->backup_sg; - TFW_DBG("sched_http: the main group is offline, use backup:" - " '%s'\n", sg->name); - srv_conn = sg->sched->sched_srv(msg, sg); - } - - if (unlikely(!srv_conn)) - TFW_DBG2("sched_http: Unable to select server from group" - " '%s'\n", sg->name); - - return srv_conn; + return tfw_sched_get_sg_srv_conn(msg, rule->main_sg, rule->backup_sg); } static TfwSrvConn * -tfw_sched_http_sched_srv(TfwMsg *msg, TfwSrvGroup *sg) +tfw_sched_http_sched_sg_conn(TfwMsg *msg, TfwSrvGroup *sg) { WARN_ONCE(true, "tfw_sched_http can't select a server from a group\n"); return NULL; } +static TfwSrvConn * +tfw_sched_http_sched_srv_conn(TfwMsg *msg, TfwServer *sg) +{ + WARN_ONCE(true, "tfw_sched_http can't select connection from a server" + "\n"); + return NULL; +} + static TfwScheduler tfw_sched_http = { .name = "http", .list = LIST_HEAD_INIT(tfw_sched_http.list), .sched_grp = tfw_sched_http_sched_grp, - .sched_srv = tfw_sched_http_sched_srv, + .sched_sg_conn = tfw_sched_http_sched_sg_conn, + .sched_srv_conn = tfw_sched_http_sched_srv_conn, }; @@ -276,6 +266,18 @@ tfw_sched_http_cfg_handle_match(TfwCfgSpec *cs, TfwCfgEntry *e) " '%s'\n", in_backup_sg); return -EINVAL; } + + /* "Default" group is not fully parsed field flag is not set. */ + if (strcasecmp(in_main_sg, "default") + && strcasecmp(in_backup_sg, "default") + && ((backup_sg->flags & TFW_SRV_STICKY_FLAGS) ^ + (main_sg->flags & TFW_SRV_STICKY_FLAGS))) + { + TFW_ERR_NL("sched_http: srv_groups '%s' and '%s' must " + "have the same sticky sessions settings\n", + in_main_sg, in_backup_sg); + return -EINVAL; + } } r = tfw_cfg_map_enum(tfw_sched_http_cfg_field_enum, in_field, &field); diff --git a/tempesta_fw/sched/tfw_sched_rr.c b/tempesta_fw/sched/tfw_sched_rr.c index 538749f50e..c6001e42d9 100644 --- a/tempesta_fw/sched/tfw_sched_rr.c +++ b/tempesta_fw/sched/tfw_sched_rr.c @@ -27,7 +27,7 @@ MODULE_AUTHOR(TFW_AUTHOR); MODULE_DESCRIPTION("Tempesta round-robin scheduler"); -MODULE_VERSION("0.2.1"); +MODULE_VERSION("0.3.0"); MODULE_LICENSE("GPL"); /** @@ -86,6 +86,7 @@ tfw_sched_rr_add_conn(TfwSrvGroup *sg, TfwServer *srv, TfwSrvConn *srv_conn) sl->srvs[s].srv = srv; ++sl->srv_n; BUG_ON(sl->srv_n > TFW_SG_MAX_SRV); + srv->sched_data = &sl->srvs[s]; } srv_cl = &sl->srvs[s]; @@ -100,6 +101,30 @@ tfw_sched_rr_add_conn(TfwSrvGroup *sg, TfwServer *srv, TfwSrvConn *srv_conn) BUG_ON(srv_cl->conn_n > TFW_SRV_MAX_CONN); } +static inline TfwSrvConn * +__sched_srv(TfwRrSrv *srv_cl, int skipnip, int *nipconn) +{ + size_t c; + + for (c = 0; c < srv_cl->conn_n; ++c) { + unsigned long idxval = atomic64_inc_return(&srv_cl->rr_counter); + TfwSrvConn *srv_conn = srv_cl->conns[idxval % srv_cl->conn_n]; + + if (unlikely(tfw_srv_conn_restricted(srv_conn) + || tfw_srv_conn_queue_full(srv_conn))) + continue; + if (skipnip && tfw_srv_conn_hasnip(srv_conn)) { + if (likely(tfw_srv_conn_live(srv_conn))) + ++(*nipconn); + continue; + } + if (likely(tfw_srv_conn_get_if_live(srv_conn))) + return srv_conn; + } + + return NULL; +} + /** * On each subsequent call the function returns the next server in the * group. Parallel connections to the same server are also rotated in @@ -118,34 +143,21 @@ tfw_sched_rr_add_conn(TfwSrvGroup *sg, TfwServer *srv, TfwSrvConn *srv_conn) * there are available server connections. */ static TfwSrvConn * -tfw_sched_rr_get_srv_conn(TfwMsg *msg, TfwSrvGroup *sg) +tfw_sched_rr_get_sg_conn(TfwMsg *msg, TfwSrvGroup *sg) { - size_t c, s; - unsigned long idxval; + size_t s; int skipnip = 1, nipconn = 0; TfwRrSrvList *sl = sg->sched_data; - TfwRrSrv *srv_cl; - TfwSrvConn *srv_conn; BUG_ON(!sl); rerun: for (s = 0; s < sl->srv_n; ++s) { - idxval = atomic64_inc_return(&sl->rr_counter); - srv_cl = &sl->srvs[idxval % sl->srv_n]; - for (c = 0; c < srv_cl->conn_n; ++c) { - idxval = atomic64_inc_return(&srv_cl->rr_counter); - srv_conn = srv_cl->conns[idxval % srv_cl->conn_n]; - if (unlikely(tfw_srv_conn_restricted(srv_conn) - || tfw_srv_conn_queue_full(srv_conn))) - continue; - if (skipnip && tfw_srv_conn_hasnip(srv_conn)) { - if (likely(tfw_srv_conn_live(srv_conn))) - nipconn++; - continue; - } - if (likely(tfw_srv_conn_get_if_live(srv_conn))) + unsigned long idxval = atomic64_inc_return(&sl->rr_counter); + TfwRrSrv *srv_cl = &sl->srvs[idxval % sl->srv_n]; + TfwSrvConn *srv_conn; + + if ((srv_conn = __sched_srv(srv_cl, skipnip, &nipconn))) return srv_conn; - } } if (skipnip && nipconn) { skipnip = 0; @@ -154,13 +166,43 @@ tfw_sched_rr_get_srv_conn(TfwMsg *msg, TfwSrvGroup *sg) return NULL; } +/** + * Same as @tfw_sched_rr_get_sg_conn(), but but schedule for a specific server + * in a group. + */ +static TfwSrvConn * +tfw_sched_rr_get_srv_conn(TfwMsg *msg, TfwServer *srv) +{ + int skipnip = 1, nipconn = 0; + TfwRrSrv *srv_cl = srv->sched_data; + TfwSrvConn *srv_conn; + + /* + * For @srv without connections srv_cl will be NULL, that normally + * does not happen in real life, but unit tests check that case. + */ + if (unlikely(!srv_cl)) + return NULL; + +rerun: + if ((srv_conn = __sched_srv(srv_cl, skipnip, &nipconn))) + return srv_conn; + + if (skipnip && nipconn) { + skipnip = 0; + goto rerun; + } + return NULL; +} + static TfwScheduler tfw_sched_rr = { .name = "round-robin", .list = LIST_HEAD_INIT(tfw_sched_rr.list), .add_grp = tfw_sched_rr_alloc_data, .del_grp = tfw_sched_rr_free_data, .add_conn = tfw_sched_rr_add_conn, - .sched_srv = tfw_sched_rr_get_srv_conn, + .sched_sg_conn = tfw_sched_rr_get_sg_conn, + .sched_srv_conn = tfw_sched_rr_get_srv_conn, }; int diff --git a/tempesta_fw/server.c b/tempesta_fw/server.c index 5ebae3c2a2..5057863a05 100644 --- a/tempesta_fw/server.c +++ b/tempesta_fw/server.c @@ -126,6 +126,7 @@ tfw_sg_new(const char *name, gfp_t flags) rwlock_init(&sg->lock); sg->sched = NULL; sg->sched_data = NULL; + sg->flags = 0; memcpy(sg->name, name, name_size); write_lock(&sg_lock); @@ -154,10 +155,10 @@ tfw_sg_free(TfwSrvGroup *sg) kfree(sg); } -int +unsigned int tfw_sg_count(void) { - int count = 0; + unsigned int count = 0; TfwSrvGroup *sg; read_lock(&sg_lock); diff --git a/tempesta_fw/server.h b/tempesta_fw/server.h index 400f053936..677f3baddf 100644 --- a/tempesta_fw/server.h +++ b/tempesta_fw/server.h @@ -37,14 +37,15 @@ typedef struct tfw_scheduler_t TfwScheduler; * * @list - member pointer in the list of servers of a server group; * @sg - back-reference to the server group; + * @sched_data - private scheduler data for the server; * @apm - opaque handle for APM stats; */ typedef struct { TFW_PEER_COMMON; struct list_head list; TfwSrvGroup *sg; + void *sched_data; void *apm; - int stress; } TfwServer; /** @@ -82,6 +83,10 @@ struct tfw_srv_group_t { /* Server related flags. */ #define TFW_SRV_RETRY_NIP 0x0001 /* Retry non-idemporent req. */ +#define TFW_SRV_STICKY_FLAGS (TFW_SRV_STICKY | TFW_SRV_STICKY_FAILOVER) +#define TFW_SRV_STICKY 0x0002 /* Use sticky sessions. */ +#define TFW_SRV_STICKY_FAILOVER 0x0004 /* Allow failovering of sticky + sessions*/ /** * Requests scheduling algorithm handler. @@ -93,18 +98,18 @@ struct tfw_srv_group_t { * @add_conn - add connection and server if it's new, called in process * context at configuration time; * @sched_grp - server group scheduling virtual method, typically returns - * result of underlying @sched_srv(); - * @sched_srv - requests scheduling virtual method, can be called in heavy - * concurrent environment; + * result of @tfw_sched_get_sg_srv_conn(); + * @sched_sg_conn - virtual method, schedules request to a server from given + * server group, returns server connection; + * @sched_srv_conn - schedule request to the given server, + * returns server connection; * - * All schedulers must be able to scheduler messages among servers of one - * server group, i.e. @sched_srv must be defined. - * However, not all the schedulers are able to designate target server group. - * If a scheduler determines server group, then it should register @sched_grp - * callback. The callback determines the target server group which references - * a scheduler responsible to distribute messages in the group. - * For the avoidance of unnecessary calls, any @sched_grp callback must call - * @sched_srv callback of the target scheduler. + * There can be 2 kind of schedulers. Tier-2 schedulers can determine + * target server connection by server or server group (@sched_srv_conn and + * @sched_sg_conn callbacks). Every server group is bound to one of the tier-2 + * schedulers. Group schedulers can find out target server group + * by message content (@sched_grp callback) and then find and outgoing + * connection by @tfw_sched_get_sg_srv_conn(). */ struct tfw_scheduler_t { const char *name; @@ -114,7 +119,8 @@ struct tfw_scheduler_t { void (*add_conn)(TfwSrvGroup *sg, TfwServer *srv, TfwSrvConn *srv_conn); TfwSrvConn *(*sched_grp)(TfwMsg *msg); - TfwSrvConn *(*sched_srv)(TfwMsg *msg, TfwSrvGroup *sg); + TfwSrvConn *(*sched_sg_conn)(TfwMsg *msg, TfwSrvGroup *sg); + TfwSrvConn *(*sched_srv_conn)(TfwMsg *msg, TfwServer *srv); }; /* Server specific routines. */ @@ -148,7 +154,7 @@ tfw_srv_conn_need_resched(TfwSrvConn *srv_conn) TfwSrvGroup *tfw_sg_lookup(const char *name); TfwSrvGroup *tfw_sg_new(const char *name, gfp_t flags); void tfw_sg_free(TfwSrvGroup *sg); -int tfw_sg_count(void); +unsigned int tfw_sg_count(void); void tfw_sg_add(TfwSrvGroup *sg, TfwServer *srv); void tfw_sg_add_conn(TfwSrvGroup *sg, TfwServer *srv, TfwSrvConn *srv_conn); @@ -158,6 +164,9 @@ void tfw_sg_release_all(void); /* Scheduler routines. */ TfwSrvConn *tfw_sched_get_srv_conn(TfwMsg *msg); +TfwSrvConn *__tfw_sched_get_srv_conn(TfwMsg *msg); +TfwSrvConn *tfw_sched_get_sg_srv_conn(TfwMsg *msg, TfwSrvGroup *main_sg, + TfwSrvGroup *backup_sg); TfwScheduler *tfw_sched_lookup(const char *name); int tfw_sched_register(TfwScheduler *sched); void tfw_sched_unregister(TfwScheduler *sched); diff --git a/tempesta_fw/sock_srv.c b/tempesta_fw/sock_srv.c index 0f88b36373..1b6f3db9c8 100644 --- a/tempesta_fw/sock_srv.c +++ b/tempesta_fw/sock_srv.c @@ -27,6 +27,7 @@ #include "tempesta_fw.h" #include "connection.h" +#include "http_sess.h" #include "addr.h" #include "log.h" #include "server.h" @@ -569,6 +570,7 @@ tfw_sock_srv_delete_all_conns(void) #define TFW_CFG_SRV_FWD_RETRIES_DEF 5 /* Default number of tries */ #define TFW_CFG_SRV_CNS_RETRIES_DEF 10 /* Reconnect tries. */ #define TFW_CFG_SRV_RETRY_NIP_DEF 0 /* Do NOT resend NIP reqs */ +#define TFW_CFG_SRV_STICKY_DEF 0 /* Don't use sticky sessions */ static TfwServer *tfw_cfg_in_slst[TFW_SG_MAX_SRV]; static TfwServer *tfw_cfg_out_slst[TFW_SG_MAX_SRV]; @@ -583,12 +585,14 @@ static int tfw_cfg_in_fwd_timeout = TFW_CFG_SRV_FWD_TIMEOUT_DEF; static int tfw_cfg_in_fwd_retries = TFW_CFG_SRV_FWD_RETRIES_DEF; static int tfw_cfg_in_cns_retries = TFW_CFG_SRV_CNS_RETRIES_DEF; static int tfw_cfg_in_retry_nip = TFW_CFG_SRV_RETRY_NIP_DEF; +static unsigned int tfw_cfg_in_sticky = TFW_CFG_SRV_STICKY_DEF; static int tfw_cfg_out_queue_size = TFW_CFG_SRV_QUEUE_SIZE_DEF; static int tfw_cfg_out_fwd_timeout = TFW_CFG_SRV_FWD_TIMEOUT_DEF; static int tfw_cfg_out_fwd_retries = TFW_CFG_SRV_FWD_RETRIES_DEF; static int tfw_cfg_out_cns_retries = TFW_CFG_SRV_CNS_RETRIES_DEF; static int tfw_cfg_out_retry_nip = TFW_CFG_SRV_RETRY_NIP_DEF; +static unsigned int tfw_cfg_out_sticky = TFW_CFG_SRV_STICKY_DEF; static int tfw_cfgop_intval(TfwCfgSpec *cs, TfwCfgEntry *ce, int *intval) @@ -659,6 +663,36 @@ tfw_cfgop_retry_nip(TfwCfgSpec *cs, TfwCfgEntry *ce, int *retry_nip) return 0; } +static inline int +tfw_cfgop_sticky(TfwCfgSpec *cs, TfwCfgEntry *ce, unsigned int *use_sticky) +{ + if (ce->attr_n) { + TFW_ERR_NL("%s: Arguments may not have the \'=\' sign\n", + cs->name); + return -EINVAL; + } + if (ce->val_n > 1) { + TFW_ERR_NL("%s: Invalid number of arguments: %zu\n", + cs->name, ce->val_n); + return -EINVAL; + } + + if (ce->val_n) { + if (!strcasecmp(ce->vals[0], "allow_failover")) { + *use_sticky |= TFW_SRV_STICKY_FAILOVER; + } + else { + TFW_ERR_NL("%s: Unsupported argument: %s\n", + cs->name, ce->vals[0]); + return -EINVAL; + } + } + + *use_sticky |= TFW_SRV_STICKY; + + return 0; +} + static int tfw_cfgop_in_retry_nip(TfwCfgSpec *cs, TfwCfgEntry *ce) { @@ -671,6 +705,18 @@ tfw_cfgop_out_retry_nip(TfwCfgSpec *cs, TfwCfgEntry *ce) return tfw_cfgop_retry_nip(cs, ce, &tfw_cfg_out_retry_nip); } +static int +tfw_cfgop_in_sticky(TfwCfgSpec *cs, TfwCfgEntry *ce) +{ + return tfw_cfgop_sticky(cs, ce, &tfw_cfg_in_sticky); +} + +static int +tfw_cfgop_out_sticky(TfwCfgSpec *cs, TfwCfgEntry *ce) +{ + return tfw_cfgop_sticky(cs, ce, &tfw_cfg_out_sticky); +} + static int tfw_cfgop_in_conn_retries(TfwCfgSpec *cs, TfwCfgEntry *ce) { @@ -889,6 +935,7 @@ tfw_cfgop_begin_srv_group(TfwCfgSpec *cs, TfwCfgEntry *ce) tfw_cfg_in_fwd_retries = tfw_cfg_out_fwd_retries; tfw_cfg_in_cns_retries = tfw_cfg_out_cns_retries; tfw_cfg_in_retry_nip = tfw_cfg_out_retry_nip; + tfw_cfg_in_sticky = tfw_cfg_out_sticky; return 0; } @@ -921,6 +968,7 @@ tfw_cfgop_finish_srv_group(TfwCfgSpec *cs) : ULONG_MAX; sg->max_refwd = tfw_cfg_in_fwd_retries ? : UINT_MAX; sg->flags |= tfw_cfg_in_retry_nip ? TFW_SRV_RETRY_NIP : 0; + sg->flags |= tfw_cfg_in_sticky; if (tfw_sg_set_sched(sg, tfw_cfg_in_sched->name)) { TFW_ERR_NL("%s %s: Unable to set scheduler: '%s'\n", @@ -1015,6 +1063,7 @@ tfw_sock_srv_start(void) : ULONG_MAX; sg->max_refwd = tfw_cfg_out_fwd_retries ? : UINT_MAX; sg->flags |= tfw_cfg_out_retry_nip ? TFW_SRV_RETRY_NIP : 0; + sg->flags |= tfw_cfg_out_sticky; if (tfw_sg_set_sched(sg, tfw_cfg_out_sched->name)) { TFW_ERR_NL("srv_group %s: Unable to set scheduler: " @@ -1100,6 +1149,13 @@ static TfwCfgSpec tfw_srv_group_specs[] = { .allow_repeat = false, .cleanup = tfw_clean_srv_groups, }, + { + "sticky_sessions", NULL, + tfw_cfgop_in_sticky, + .allow_none = true, + .allow_repeat = false, + .cleanup = tfw_clean_srv_groups, + }, { 0 } }; @@ -1157,6 +1213,13 @@ TfwCfgMod tfw_sock_srv_cfg_mod = { .allow_repeat = true, .cleanup = tfw_clean_srv_groups, }, + { + "sticky_sessions", NULL, + tfw_cfgop_out_sticky, + .allow_none = true, + .allow_repeat = false, + .cleanup = tfw_clean_srv_groups, + }, { "srv_group", NULL, tfw_cfg_handle_children, diff --git a/tempesta_fw/t/functional/helpers/control.py b/tempesta_fw/t/functional/helpers/control.py index d7f2d2deea..60286acefd 100644 --- a/tempesta_fw/t/functional/helpers/control.py +++ b/tempesta_fw/t/functional/helpers/control.py @@ -235,6 +235,23 @@ def clients_run_parallel(clients): pool.map(__clients_parse_output, parse_args) pool.map(__clients_cleanup, clients) +def clients_parallel_load(client, count=None): + """ Spawn @count processes without parsing output. Just make high load. + Python is too slow to spawn multiple (>100) processes. + """ + if count is None: + count = min(int(tf_cfg.cfg.get('General', 'concurrent_connections')), 1000) + tf_cfg.dbg(3, ('\tRunning %d HTTP clients on %s' % + (count, remote.client.host))) + error.assertTrue(client.prepare()) + cmd = 'seq %d | xargs -Iz -P1000 %s -q' % (count, client.cmd) + + pool = multiprocessing.Pool(2) + results = pool.map(remote.client.run_cmd, [client.cmd, cmd]) + + stdout, stderr = results[0] + client.parse_out(stdout, stderr) + client.cleanup() #------------------------------------------------------------------------------- # Tempesta diff --git a/tempesta_fw/t/functional/helpers/deproxy.py b/tempesta_fw/t/functional/helpers/deproxy.py index ad149a09ba..55d549060a 100644 --- a/tempesta_fw/t/functional/helpers/deproxy.py +++ b/tempesta_fw/t/functional/helpers/deproxy.py @@ -345,7 +345,7 @@ def parse_firstline(self, stream): words = statusline.rstrip('\r\n').split() if len(words) >= 3: self.version, self.status = words[0:2] - self.reason = words[2:] + self.reason = ' '.join(words[2:]) elif len(words) == 2: self.version, self.status = words else: @@ -420,7 +420,7 @@ def handle_read(self): self.response_buffer += self.recv(MAX_MESSAGE_SIZE) if not self.response_buffer: return - tf_cfg.dbg(4, '\tDeproxy: Client: Recieve response from server.') + tf_cfg.dbg(4, '\tDeproxy: Client: Recieve response from Tempesta.') tf_cfg.dbg(5, self.response_buffer) try: response = Response(self.response_buffer) @@ -461,7 +461,7 @@ def __init__(self, tester, server, sock=None, keep_alive=None): self.responses_done = 0 self.request_buffer = '' self.tester.register_srv_connection(self) - tf_cfg.dbg(4, '\tDeproxy: SrvConnection: New server connection.') + tf_cfg.dbg(6, '\tDeproxy: SrvConnection: New server connection.') def handle_read(self): self.request_buffer += self.recv(MAX_MESSAGE_SIZE) @@ -500,34 +500,55 @@ def handle_error(self): error.bug('\tDeproxy: SrvConnection: %s' % v) def handle_close(self): + tf_cfg.dbg(6, '\tDeproxy: SrvConnection: Close connection.') + self.close() if self.tester: self.tester.remove_srv_connection(self) - asyncore.dispatcher_with_send.handle_close(self) - - def close(self): - tf_cfg.dbg(4, '\tDeproxy: SrvConnection: Close connection.') - asyncore.dispatcher_with_send.close(self) + if self.server: + try: + self.server.connections.remove(self) + except: + pass class Server(asyncore.dispatcher): - def __init__(self, port, host=None, connections=None, keep_alive=None): + def __init__(self, port, host=None, conns_n=None, keep_alive=None): asyncore.dispatcher.__init__(self) self.tester = None self.port = port - if connections is None: - connections = tempesta.server_conns_default() - self.conns_n = connections + self.connections = [] + if conns_n is None: + conns_n = tempesta.server_conns_default() + self.conns_n = conns_n self.keep_alive = keep_alive if host is None: host = 'Client' - addr = tf_cfg.cfg.get('Client', 'ip') - tf_cfg.dbg(4, '\tDeproxy: Server: Start on %s:%d.' % (addr, port)) + self.addr_str = tf_cfg.cfg.get('Client', 'ip') + tf_cfg.dbg(4, '\tDeproxy: Server: Start on %s:%d.' % (self.addr_str, port)) + self.setup() + + def setup(self): self.create_socket(socket.AF_INET, socket.SOCK_STREAM) self.set_reuse_addr() - self.bind((addr, port)) + self.bind((self.addr_str, self.port)) self.listen(socket.SOMAXCONN) + def restart(self): + asyncore.dispatcher.__init__(self) + self.tester.servers.append(self) + self.setup() + + def stop(self): + tf_cfg.dbg(4, '\tDeproxy: Server: Stop on %s:%d.' % (self.addr_str, + self.port)) + self.close() + connections = [conn for conn in self.connections] + for conn in connections: + conn.handle_close() + if self.tester: + self.tester.servers.remove(self) + def set_tester(self, tester): self.tester = tester @@ -537,10 +558,20 @@ def handle_accept(self): sock, addr = pair handler = ServerConnection(self.tester, server=self, sock=sock, keep_alive=self.keep_alive) + self.connections.append(handler) + assert len(self.connections) <= self.conns_n, \ + ('Too lot connections, expect %d, got %d' + & (self.conns_n, len(self.connections))) + + def active_conns_n(self): + return len(self.connections) def handle_error(self): t, v, tb = sys.exc_info() - error.bug('\tDeproxy: Server: %s' % v) + error.bug('\tDeproxy: Server %s:%d: %s' % (self.addr_str, self.port, v)) + + def handle_close(self): + self.stop() #------------------------------------------------------------------------------- @@ -639,6 +670,8 @@ def recieved_forwarded_request(self, request, connection): return self.current_chain.server_response def register_srv_connection(self, connection): + assert connection.server in self.servers, \ + 'Register connection, which comes from not registred server!' self.srv_connections.append(connection) def remove_srv_connection(self, connection): @@ -651,11 +684,12 @@ def remove_srv_connection(self, connection): def is_srvs_ready(self): expected_conns_n = sum([s.conns_n for s in self.servers]) + assert len(self.srv_connections) <= expected_conns_n, \ + 'Registered more connections that must be!.' return expected_conns_n == len(self.srv_connections) def close_all(self): - self.client.close() - for conn in self.srv_connections: - conn.close() - for server in self.servers: - server.close() + self.client.handle_close() + servers = [server for server in self.servers] + for server in servers: + server.handle_close() diff --git a/tempesta_fw/t/functional/helpers/siege.py b/tempesta_fw/t/functional/helpers/siege.py index 292e375695..eed05be184 100644 --- a/tempesta_fw/t/functional/helpers/siege.py +++ b/tempesta_fw/t/functional/helpers/siege.py @@ -16,6 +16,8 @@ def __init__(self): self.options = [ # Default concurrent. ('concurrent', '25'), + # Limit of threads. + ('limit', '255'), # Disable printing eash transaction to stdout. ('verbose', 'false'), # Leave color for humas. It breaks regexes. diff --git a/tempesta_fw/t/functional/helpers/tempesta.py b/tempesta_fw/t/functional/helpers/tempesta.py index 9a3a3eac27..bfdecf2eb7 100644 --- a/tempesta_fw/t/functional/helpers/tempesta.py +++ b/tempesta_fw/t/functional/helpers/tempesta.py @@ -140,6 +140,8 @@ def __init__(self, name='default', sched='round-robin'): self.name = name self.sched = sched self.servers = [] + # Server group options, isserted after servers. + self.options = '' def add_server(self, ip, port, conns=server_conns_default()): error.assertTrue(conns <= server_conns_max()) @@ -151,11 +153,12 @@ def add_server(self, ip, port, conns=server_conns_default()): def get_config(self): sg = '' if self.name == 'default': - sg = '\n'.join(['sched %s;' % self.sched] + self.servers) + sg = '\n'.join(['sched %s;' % self.sched] + self.servers + + [self.options]) else: sg = '\n'.join( ['srv_group %s {' % self.name] + ['sched %s;' % self.sched] + - self.servers + ['}']) + self.servers + [self.options] + ['}']) return sg class Config(object): diff --git a/tempesta_fw/t/functional/regression/test_shutdown.py b/tempesta_fw/t/functional/regression/test_shutdown.py index 9897249982..758c858e5f 100644 --- a/tempesta_fw/t/functional/regression/test_shutdown.py +++ b/tempesta_fw/t/functional/regression/test_shutdown.py @@ -25,10 +25,10 @@ def setUp(self): self.clients = [] def tearDown(self): - if self.tester: - self.tester.close_all() if self.tempesta: self.tempesta.stop() + if self.tester: + self.tester.close_all() def create_client(self): for i in range(100): @@ -99,8 +99,7 @@ def run(self): def close_all(self): for client in self.clients: - client.close() - for conn in self.srv_connections: - conn.close() - for server in self.servers: - server.close() + client.handle_close() + servers = [server for server in self.servers] + for server in servers: + server.handle_close() diff --git a/tempesta_fw/t/functional/sched/test_http.py b/tempesta_fw/t/functional/sched/test_http.py index 1c90cb3257..951abf7cb9 100644 --- a/tempesta_fw/t/functional/sched/test_http.py +++ b/tempesta_fw/t/functional/sched/test_http.py @@ -61,7 +61,7 @@ def create_servers(self): for group, uri, header, value in server_options: # Dont need too lot connections here. - server = deproxy.Server(port=port, connections=1) + server = deproxy.Server(port=port, conns_n=1) port += 1 server.group = group server.chains = self.make_chains(uri=uri, @@ -148,7 +148,7 @@ def make_chains(self, empty=True): return [chain for i in range(self.requests_n)] def create_server_helper(self, group, port): - server = deproxy.Server(port=port, connections=1) + server = deproxy.Server(port=port, conns_n=1) server.group = group server.chains = self.make_chains() return server diff --git a/tempesta_fw/t/functional/sched/test_rr.py b/tempesta_fw/t/functional/sched/test_rr.py index f81a9f1ced..d6ca873473 100644 --- a/tempesta_fw/t/functional/sched/test_rr.py +++ b/tempesta_fw/t/functional/sched/test_rr.py @@ -8,6 +8,7 @@ import unittest import math import random +import sys from helpers import tempesta from testers import stress @@ -15,19 +16,28 @@ __copyright__ = 'Copyright (C) 2017 Tempesta Technologies, Inc.' __license__ = 'GPL2' +class RrStressTest(stress.StressTest): + """Stress test for RR scheduler: max servers in group, default keep-alive + settings. Clients must not recieve non-2xx answers. + """ + + def create_servers(self): + self.create_servers_helper(tempesta.servers_in_group()) + + def test_rr(self): + self.generic_test_routine('cache 0;\n') -class FairLoadEqualConns(stress.StressTest): + +class FairLoadEqualConns(RrStressTest): """ Round-Robin scheduler loads all the upstream servers in the fair - way. In this test servers have the same connections count. - """ + way. In this test servers have the same connections count. + """ # Precision of fair loading. - precision = 0.02 - - def create_servers(self): - self.create_servers_helper(tempesta.servers_in_group()) + precision = 0.005 def assert_servers(self): + """All servers must recieve almost equal amount of requests.""" self.servers_get_stats() cl_reqs = self.tempesta.stats.cl_msg_forwarded s_reqs_expected = cl_reqs / len(self.servers) @@ -39,6 +49,9 @@ def assert_servers(self): self.assertEqual(s_reqs, self.tempesta.stats.cl_msg_forwarded) def test_rr(self): + # Server connections failovering may affect load distribution. + for s in self.servers: + s.config.set_ka(sys.maxsize) self.generic_test_routine('cache 0;\n') diff --git a/tempesta_fw/t/functional/selftests/test_deproxy.py b/tempesta_fw/t/functional/selftests/test_deproxy.py index 6e353c5d64..2ef1ae4c6f 100644 --- a/tempesta_fw/t/functional/selftests/test_deproxy.py +++ b/tempesta_fw/t/functional/selftests/test_deproxy.py @@ -23,6 +23,7 @@ class DeproxyDummyTest(functional.FunctionalTest): def setUp(self): self.client = None self.servers = [] + self.tester = None tf_cfg.dbg(3) # Step to the next line after name of test case. tf_cfg.dbg(3, '\tInit test case...') @@ -38,7 +39,7 @@ def create_clients(self): def create_servers(self): port = tempesta.upstream_port_start_from() - self.servers = [deproxy.Server(port=port, connections=1)] + self.servers = [deproxy.Server(port=port, conns_n=1)] def routine(self, message_chains): self.create_servers() diff --git a/tempesta_fw/t/functional/sessions/cookies.py b/tempesta_fw/t/functional/sessions/cookies.py new file mode 100644 index 0000000000..1e16fd99b3 --- /dev/null +++ b/tempesta_fw/t/functional/sessions/cookies.py @@ -0,0 +1,125 @@ +from __future__ import print_function +import re +from helpers import deproxy, tf_cfg +from testers import functional + +__author__ = 'Tempesta Technologies, Inc.' +__copyright__ = 'Copyright (C) 2017 Tempesta Technologies, Inc.' +__license__ = 'GPL2' + +CHAIN_LENGTH = 20 + +def chains(): + chain = functional.base_message_chain() + return [chain for i in range(CHAIN_LENGTH)] + +def make_302(request): + response = deproxy.Response( + 'HTTP/1.1 302 Found\r\n' + 'Content-Length: 0\r\n' + 'Location: http://%s%s\r\n' + 'Connection: keep-alive\r\n' + '\r\n' + % (tf_cfg.cfg.get('Tempesta', 'ip'), request.uri)) + return response + +def make_502(): + response = deproxy.Response( + 'HTTP/1.1 502 Bad Gateway\r\n' + 'Content-Length: 0\r\n' + 'Connection: keep-alive\r\n' + '\r\n') + return response + +class TesterIgnoreCookies(deproxy.Deproxy): + """Tester helper. Emulate client that does not support cookies.""" + + def __init__(self, *args, **kwargs): + deproxy.Deproxy.__init__(self, *args, **kwargs) + self.message_chains = chains() + self.cookies = [] + + def recieved_response(self, response): + m = re.search(r'__tfw=([a-f0-9]+)', response.headers['Set-Cookie']) + assert m, 'Set-Cookie header not found!' + cookie = m.group(1) + + # Tempesta sent us a Cookie, and we were waiting for it. + exp_resp = self.current_chain.response + exp_resp.headers.delete_all('Set-Cookie') + exp_resp.headers.add('Set-Cookie', response.headers['Set-Cookie']) + exp_resp.update() + + # Client doesn't support cookies: Tempesta will generate new cookie for + # each request. + assert cookie not in self.cookies, \ + 'Recieved non-uniquee cookie!' + + if exp_resp.status != '200': + exp_resp.headers.delete_all('Date') + exp_resp.headers.add('Date', response.headers['Date']) + exp_resp.update() + + deproxy.Deproxy.recieved_response(self, response) + + +class TesterIgnoreEnforcedCookies(TesterIgnoreCookies): + """Tester helper. Emulate client that does not support cookies, but + Tempesta enforces cookies. + """ + + def __init__(self, *args, **kwargs): + TesterIgnoreCookies.__init__(self, *args, **kwargs) + self.message_chains[0].response = make_302( + self.message_chains[0].request) + self.message_chains[0].server_response = deproxy.Response() + self.message_chains[0].fwd_request = deproxy.Request() + + +class TesterUseCookies(deproxy.Deproxy): + """Tester helper. Emulate client that support cookies.""" + + def __init__(self, *args, **kwargs): + deproxy.Deproxy.__init__(self, *args, **kwargs) + # The first message chain is unique. + self.message_chains = [functional.base_message_chain()] + chains() + self.cookie_parsed = False + + def recieved_response(self, response): + if not self.cookie_parsed: + m = re.search(r'__tfw=([a-f0-9]+)', response.headers['Set-Cookie']) + assert m, 'Set-Cookie header not found!' + cookie = m.group(1) + + # Tempesta sent us a Cookie, and we was waiting for it. + exp_resp = self.current_chain.response + exp_resp.headers.delete_all('Set-Cookie') + exp_resp.headers.add('Set-Cookie', response.headers['Set-Cookie']) + exp_resp.update() + + # All folowing requests must contain Cookie header + for req in [self.message_chains[1].request, + self.message_chains[1].fwd_request]: + req.headers.add('Cookie', ''.join(['__tfw=', cookie])) + req.update() + + self.cookie_parsed = True + + exp_resp = self.current_chain.response + if exp_resp.status != '200': + exp_resp.headers.delete_all('Date') + exp_resp.headers.add('Date', response.headers['Date']) + exp_resp.update() + + deproxy.Deproxy.recieved_response(self, response) + + +class TesterUseEnforcedCookies(TesterUseCookies): + """Tester helper. Emulate client that support cookies.""" + + def __init__(self, *args, **kwargs): + TesterUseCookies.__init__(self, *args, **kwargs) + self.message_chains[0].response = make_302( + self.message_chains[0].request) + self.message_chains[0].server_response = deproxy.Response() + self.message_chains[0].fwd_request = deproxy.Request() diff --git a/tempesta_fw/t/functional/sessions/test_cookies.py b/tempesta_fw/t/functional/sessions/test_cookies.py index f120f1e763..3d5e0f02de 100644 --- a/tempesta_fw/t/functional/sessions/test_cookies.py +++ b/tempesta_fw/t/functional/sessions/test_cookies.py @@ -1,63 +1,55 @@ from __future__ import print_function import sys -from helpers import tf_cfg, control -from testers import stress +import unittest +import re +from helpers import deproxy +from testers import functional +from . import cookies __author__ = 'Tempesta Technologies, Inc.' __copyright__ = 'Copyright (C) 2017 Tempesta Technologies, Inc.' __license__ = 'GPL2' -config_cookies = """ -cache 0; -sticky; -sticky_secret "f00)9eR59*_/22"; -sess_lifetime 100; -""" -config_cookies_enforced = """ -cache 0; -sticky enforce; -sticky_secret "f00)9eR59*_/22"; -sess_lifetime 100; -""" - -# UserAgent headers example, id must be filled before using -ua_example = 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:47.0; id:%d) Gecko/20100101 Firefox/47.0' - -class StressCookies(stress.StressTest): - """ Stress test for cookies. Clients do not support cookies. """ - - def create_clients_helper(self, client_class): - """ Cookies depends on IP adress and UserAgent header. We cannot affect - ip in the test, but we can start several traffic generators with unique - UserAgent headers each instead. - """ - self.clients = [] - conns = int(tf_cfg.cfg.get('General', 'concurrent_connections')) - for i in range(conns): - client = client_class() - client.set_user_agent(ua_example % i) - client.connections = 1 - self.clients.append(client) - - def create_clients(self): - self.create_clients_helper(control.Wrk) - - def test_cookies(self): - # FIXME: #383 workaround - for s in self.servers: - s.config.set_ka(sys.maxsize) - self.generic_test_routine(config_cookies) - - -class StressEnforcedCookies(StressCookies): - """ Stress test for cookies. Clients support cookies. Cookies are enforced. - """ - - def create_clients(self): - self.create_clients_helper(control.Siege) - - def test_cookies(self): - # FIXME: #383 workaround - for s in self.servers: - s.config.set_ka(sys.maxsize) - self.generic_test_routine(config_cookies_enforced) + +class TestNoCookiesSupport(functional.FunctionalTest): + """ Functional test for using cookies cookie. """ + + config = ( + 'cache 0;\n' + 'sticky;\n' + 'sticky_secret "f00)9eR59*_/22";\n' + '\n') + + def create_tester(self, message_chain): + self.tester = cookies.TesterIgnoreCookies(message_chain, self.client, + self.servers) + + def test(self): + self.generic_test_routine(self.config, []) + + +class TestCookiesSupport(TestNoCookiesSupport): + + def create_tester(self, message_chain): + self.tester = cookies.TesterUseCookies(message_chain, self.client, + self.servers) + + +class TestNoEnforcedCookiesSupport(TestNoCookiesSupport): + + config = ( + 'cache 0;\n' + 'sticky enforce;\n' + 'sticky_secret "f00)9eR59*_/22";\n' + '\n') + + def create_tester(self, message_chain): + self.tester = cookies.TesterIgnoreEnforcedCookies( + message_chain, self.client, self.servers) + + +class TestEnforcedCookiesSupport(TestNoEnforcedCookiesSupport): + + def create_tester(self, message_chain): + self.tester = cookies.TesterUseEnforcedCookies( + message_chain, self.client, self.servers) diff --git a/tempesta_fw/t/functional/sessions/test_sticky_sess.py b/tempesta_fw/t/functional/sessions/test_sticky_sess.py new file mode 100644 index 0000000000..10816576de --- /dev/null +++ b/tempesta_fw/t/functional/sessions/test_sticky_sess.py @@ -0,0 +1,152 @@ +from __future__ import print_function +import sys +import unittest +import re +import copy +from helpers import deproxy, tempesta +from testers import functional +from . import cookies + +__author__ = 'Tempesta Technologies, Inc.' +__copyright__ = 'Copyright (C) 2017 Tempesta Technologies, Inc.' +__license__ = 'GPL2' + +defconfig = ( + 'cache 0;\n' + 'sticky %s;\n' + 'sticky_secret "f00)9eR59*_/22";\n' + '\n') + +class TestSticky(functional.FunctionalTest): + """ Functional test for using sticky sessions. """ + + # No enforce + config = defconfig % '' + + def create_servers(self): + self.create_servers_helper(tempesta.servers_in_group(), + connections=1) + def configure_tempesta(self): + functional.FunctionalTest.configure_tempesta(self) + sg = self.tempesta.config.server_groups[0] + if self.allow_failover: + sg.options = 'sticky_sessions allow_failover;' + else: + sg.options = 'sticky_sessions;' + + def create_tester(self, message_chain): + self.tester = TesterSticky(message_chain, self.client, self.servers) + + def chain_failover_ok(self): + return self.tester.message_chains[1:] + + def chain_failover_fobbiden(self): + chain = copy.copy(self.tester.message_chains[1]) + chain.no_forward() + chain.response = cookies.make_502() + return [chain for i in range(cookies.CHAIN_LENGTH)] + + def check_failover(self, new_message_chain_provider): + self.generic_test_routine(self.config, []) + message_chains = self.tester.message_chains + # Shutdown server pinned to session and all its connections. + self.previous_srv = self.tester.pinned_srv + self.previous_srv.handle_close() + self.tester.pinned_srv = None + self.tester.used_srv = None + # Set new message chain after shutdown: + self.tester.message_chains = new_message_chain_provider() + self.tester.run() + assert not self.tester.pinned_srv is self.previous_srv, \ + 'Sticky session is forwarded to offline server' + # Restore original message chains + self.tester.message_chains = message_chains + + def check_back_online(self, new_message_chain_provider, restore): + self.check_failover(new_message_chain_provider) + # Return previous server back online. + self.previous_srv.restart() + + if restore: + self.tester.pinned_srv = self.previous_srv + # Remove cookie negotiation chain, reuse old cookie. + self.tester.message_chains = self.tester.message_chains[1:] + else: + self.tester.message_chains = new_message_chain_provider() + self.tester.run() + + def test(self): + """Simply sticky connections.""" + self.allow_failover = False + self.generic_test_routine(self.config, []) + + def test_failover_fobbiden(self): + """No Failover: if pinned server goes donw, return 502.""" + self.allow_failover = False + self.check_failover(self.chain_failover_fobbiden) + + def test_failover(self): + """With Failover: if pinned server goes down use new one.""" + self.allow_failover = True + self.check_failover(self.chain_failover_ok) + + def test_back_online_no_failover(self): + """No Failover: continue use pinned server if it back online.""" + self.allow_failover = False + self.check_back_online(self.chain_failover_fobbiden, True) + + def test_back_online_after_failover(self): + """With Failover: even if original server returned back proceed with + the replacement server.""" + self.allow_failover = True + self.check_back_online(self.chain_failover_ok, False) + + +class TestStickyEnforcedCookies(TestSticky): + """ Functional test for using sticky sessions, cookies are enforced. """ + # Enforce + config = defconfig % 'enforce' + + def create_tester(self, message_chain): + self.tester = TesterStickyEnforcedCookies( + message_chain, self.client, self.servers) + + +class TesterSticky(cookies.TesterUseCookies): + + def __init__(self, *args, **kwargs): + cookies.TesterUseCookies.__init__(self, *args, **kwargs) + self.pinned_srv = None + self.used_srv = None + + def recieved_forwarded_request(self, request, connection): + if not self.pinned_srv: + self.pinned_srv = connection.server + self.used_srv = connection.server + return cookies.TesterUseCookies.recieved_forwarded_request( + self, request, connection) + + def check_expectations(self): + cookies.TesterUseCookies.check_expectations(self) + assert self.pinned_srv is self.used_srv, \ + 'Session is not Sticky, request forwarded to other server!' + + +class TesterStickyEnforcedCookies(cookies.TesterUseEnforcedCookies): + + def __init__(self, *args, **kwargs): + cookies.TesterUseEnforcedCookies.__init__(self, *args, **kwargs) + self.pinned_srv = None + self.used_srv = None + + def recieved_forwarded_request(self, request, connection): + if not self.pinned_srv: + self.pinned_srv = connection.server + self.used_srv = connection.server + return cookies.TesterUseEnforcedCookies.recieved_forwarded_request( + self, request, connection) + + def check_expectations(self): + cookies.TesterUseCookies.check_expectations(self) + assert self.pinned_srv is self.used_srv, \ + 'Session is not Sticky, request forwarded to other server!' diff --git a/tempesta_fw/t/functional/sessions/test_sticky_sess_stress.py b/tempesta_fw/t/functional/sessions/test_sticky_sess_stress.py new file mode 100644 index 0000000000..c3bb99ce02 --- /dev/null +++ b/tempesta_fw/t/functional/sessions/test_sticky_sess_stress.py @@ -0,0 +1,74 @@ +""" +With sticky sessions each client is pinned to only one server in group. +""" + +from __future__ import print_function +import unittest, sys +from helpers import control, tempesta, tf_cfg +from testers import stress + +__author__ = 'Tempesta Technologies, Inc.' +__copyright__ = 'Copyright (C) 2017 Tempesta Technologies, Inc.' +__license__ = 'GPL2' + +class OneClient(stress.StressTest): + + config = ( + 'cache 0;\n' + 'sticky enforce;\n' + 'sticky_secret "f00)9eR59*_/22";\n' + '\n') + + def configure_tempesta(self): + stress.StressTest.configure_tempesta(self) + for sg in self.tempesta.config.server_groups: + sg.options = 'sticky_sessions;' + + def create_clients(self): + siege = control.Siege() + siege.rc.set_option('limit', '4096') + siege.rc.set_option('connection', 'keep-alive') + siege.rc.set_option('parser', 'false') + self.clients = [siege] + + def create_servers(self): + self.create_servers_helper(tempesta.servers_in_group()) + + def assert_servers(self): + self.servers_get_stats() + expected_err = int(tf_cfg.cfg.get('General', 'concurrent_connections')) + exp_min = self.clients[0].requests - expected_err - 1 + exp_max = self.clients[0].requests + expected_err + 1 + # Only one server must pull all the load. + loaded = 0 + for s in self.servers: + if s.requests: + loaded += 1 + self.assertTrue(s.requests in range(exp_min, exp_max)) + self.assertEqual(loaded, 1) + + def test(self): + self.generic_test_routine(self.config) + + +class LotOfClients(OneClient): + + def create_clients(self): + # Don't use client array here. Too slow for running + self.siege = control.Siege() + self.siege.rc.set_option('connection', 'keep-alive') + self.siege.rc.set_option('parser', 'false') + self.siege.connections = 25 + self.clients = [self.siege] + + def test(self): + self.tempesta.config.set_defconfig(self.config) + self.configure_tempesta() + control.servers_start(self.servers) + self.tempesta.start() + + control.clients_parallel_load(self.siege) + + self.tempesta.get_stats() + self.assert_clients() + self.assert_tempesta() diff --git a/tempesta_fw/t/functional/testers/functional.py b/tempesta_fw/t/functional/testers/functional.py index b031b9c24a..911bb47ec5 100644 --- a/tempesta_fw/t/functional/testers/functional.py +++ b/tempesta_fw/t/functional/testers/functional.py @@ -1,6 +1,7 @@ from __future__ import print_function import unittest import copy +import asyncore from helpers import tf_cfg, control, tempesta, deproxy __author__ = 'Tempesta Technologies, Inc.' @@ -33,7 +34,8 @@ def create_servers(self): port = tempesta.upstream_port_start_from() self.servers = [deproxy.Server(port=port)] - def create_servers_helper(self, count, start_port=None, keep_alive=None): + def create_servers_helper(self, count, start_port=None, keep_alive=None, + connections=None): """ Helper function to spawn `count` servers in default configuration. """ if start_port is None: @@ -41,7 +43,8 @@ def create_servers_helper(self, count, start_port=None, keep_alive=None): self.servers = [] for i in range(count): self.servers.append(deproxy.Server(port=(start_port + i), - keep_alive=keep_alive)) + keep_alive=keep_alive, + conns_n=connections)) def setUp(self): self.client = None @@ -54,6 +57,7 @@ def setUp(self): def tearDown(self): # Close client connection before stopping the TempestaFW. + asyncore.close_all() if self.client: self.client.close() if self.tempesta: @@ -61,6 +65,10 @@ def tearDown(self): if self.tester: self.tester.close_all() + @classmethod + def tearDownClass(cls): + asyncore.close_all() + def assert_tempesta(self): """ Assert that tempesta had no errors during test. """ msg = 'Tempesta have errors in processing HTTP %s.' diff --git a/tempesta_fw/t/unit/sched_helper.c b/tempesta_fw/t/unit/sched_helper.c index d6efc6075b..0bd4366d41 100644 --- a/tempesta_fw/t/unit/sched_helper.c +++ b/tempesta_fw/t/unit/sched_helper.c @@ -141,8 +141,11 @@ test_conn_release_all(TfwSrvGroup *sg) } } +/** + * Unit test. Message cannot be scheduled to empty server group. + */ void -test_sched_generic_empty_sg(struct TestSchedHelper *sched_helper) +test_sched_sg_empty_sg(struct TestSchedHelper *sched_helper) { size_t i; TfwSrvGroup *sg; @@ -157,7 +160,7 @@ test_sched_generic_empty_sg(struct TestSchedHelper *sched_helper) for (i = 0; i < sched_helper->conn_types; ++i) { TfwMsg *msg = sched_helper->get_sched_arg(i); - TfwSrvConn *srv_conn = sg->sched->sched_srv(msg, sg); + TfwSrvConn *srv_conn = sg->sched->sched_sg_conn(msg, sg); EXPECT_NULL(srv_conn); sched_helper->free_sched_arg(msg); @@ -166,8 +169,12 @@ test_sched_generic_empty_sg(struct TestSchedHelper *sched_helper) test_sg_release_all(); } +/** + * Unit test. Message cannot be scheduled to server group if server in that + * group have no live connections. + */ void -test_sched_generic_one_srv_zero_conn(struct TestSchedHelper *sched_helper) +test_sched_sg_one_srv_zero_conn(struct TestSchedHelper *sched_helper) { size_t i; TfwSrvGroup *sg; @@ -184,7 +191,7 @@ test_sched_generic_one_srv_zero_conn(struct TestSchedHelper *sched_helper) for (i = 0; i < sched_helper->conn_types; ++i) { TfwMsg *msg = sched_helper->get_sched_arg(i); - TfwSrvConn *srv_conn = sg->sched->sched_srv(msg, sg); + TfwSrvConn *srv_conn = sg->sched->sched_sg_conn(msg, sg); EXPECT_NULL(srv_conn); sched_helper->free_sched_arg(msg); @@ -193,8 +200,13 @@ test_sched_generic_one_srv_zero_conn(struct TestSchedHelper *sched_helper) test_sg_release_all(); } +/** + * Unit test. Message cannot be scheduled to server group if servers in that + * group have no live connections. Server group contain as much servers as + * possible. + */ void -test_sched_generic_max_srv_zero_conn(struct TestSchedHelper *sched_helper) +test_sched_sg_max_srv_zero_conn(struct TestSchedHelper *sched_helper) { size_t i, j; TfwSrvGroup *sg; @@ -211,14 +223,157 @@ test_sched_generic_max_srv_zero_conn(struct TestSchedHelper *sched_helper) test_create_srv("127.0.0.1", sg); for (i = 0; i < sched_helper->conn_types; ++i) { + TfwMsg *msg = sched_helper->get_sched_arg(i); + for (j = 0; j < TFW_SG_MAX_SRV; ++j) { - TfwMsg *msg = sched_helper->get_sched_arg(i); - TfwSrvConn *srv_conn = sg->sched->sched_srv(msg, sg); + TfwSrvConn *srv_conn = + sg->sched->sched_sg_conn(msg, sg); + + EXPECT_NULL(srv_conn); + /* + * Don't let wachtdog wuppose that we have stucked + * on long cycles. + */ + kernel_fpu_end(); + schedule(); + kernel_fpu_begin(); + } + sched_helper->free_sched_arg(msg); + } + + test_sg_release_all(); +} + +/** + * Unit test. Message cannot be scheduled to server if it has no live + * connections. + */ +void +test_sched_srv_one_srv_zero_conn(struct TestSchedHelper *sched_helper) +{ + size_t i; + TfwSrvGroup *sg; + TfwServer *srv; + + BUG_ON(!sched_helper); + BUG_ON(!sched_helper->sched); + BUG_ON(!sched_helper->conn_types); + BUG_ON(!sched_helper->get_sched_arg); + BUG_ON(!sched_helper->free_sched_arg); + + sg = test_create_sg("test", sched_helper->sched); + + srv = test_create_srv("127.0.0.1", sg); + + for (i = 0; i < sched_helper->conn_types; ++i) { + TfwMsg *msg = sched_helper->get_sched_arg(i); + TfwSrvConn *srv_conn = sg->sched->sched_srv_conn(msg, srv); + + EXPECT_NULL(srv_conn); + sched_helper->free_sched_arg(msg); + } + + test_sg_release_all(); +} + +/** + * Unit test. Message cannot be scheduled to any server of server group if + * there is no no live connections across all server. + */ +void +test_sched_srv_max_srv_zero_conn(struct TestSchedHelper *sched_helper) +{ + size_t i, j; + TfwSrvGroup *sg; + + BUG_ON(!sched_helper); + BUG_ON(!sched_helper->sched); + BUG_ON(!sched_helper->conn_types); + BUG_ON(!sched_helper->get_sched_arg); + BUG_ON(!sched_helper->free_sched_arg); + + sg = test_create_sg("test", sched_helper->sched); + + for (j = 0; j < TFW_SG_MAX_SRV; ++j) + test_create_srv("127.0.0.1", sg); + + for (i = 0; i < sched_helper->conn_types; ++i) { + TfwMsg *msg = sched_helper->get_sched_arg(i); + TfwServer *srv; + + list_for_each_entry(srv, &sg->srv_list, list) { + TfwSrvConn *srv_conn = + sg->sched->sched_srv_conn(msg, srv); EXPECT_NULL(srv_conn); - sched_helper->free_sched_arg(msg); + /* + * Don't let wachtdog wuppose that we have stucked + * on long cycles. + */ + kernel_fpu_end(); + schedule(); + kernel_fpu_begin(); } + sched_helper->free_sched_arg(msg); + } + + test_sg_release_all(); +} + +/** + * Unit test. Message cannot be scheduled to server if it is in failovering + * process. + */ +void +test_sched_srv_offline_srv(struct TestSchedHelper *sched_helper) +{ + size_t i; + size_t offline_num = 3; + TfwServer *offline_srv = NULL; + TfwSrvGroup *sg; + + BUG_ON(!sched_helper); + BUG_ON(!sched_helper->sched); + BUG_ON(!sched_helper->conn_types); + BUG_ON(!sched_helper->get_sched_arg); + BUG_ON(!sched_helper->free_sched_arg); + BUG_ON(offline_num >= TFW_SG_MAX_SRV); + + sg = test_create_sg("test", sched_helper->sched); + + for (i = 0; i < TFW_SG_MAX_SRV; ++i) { + TfwServer *srv = test_create_srv("127.0.0.1", sg); + TfwSrvConn *srv_conn = test_create_conn((TfwPeer *)srv); + sg->sched->add_conn(sg, srv, srv_conn); + + if (i == offline_num) { + offline_srv = srv; + atomic_set(&srv_conn->refcnt, 0); + } + } + + for (i = 0; i < sched_helper->conn_types; ++i) { + TfwMsg *msg = sched_helper->get_sched_arg(i); + TfwServer *srv; + list_for_each_entry(srv, &sg->srv_list, list) { + TfwSrvConn *srv_conn = + sg->sched->sched_srv_conn(msg, srv); + + if (srv == offline_srv) + EXPECT_NULL(srv_conn); + else + EXPECT_NOT_NULL(srv_conn); + /* + * Don't let wachtdog wuppose that we have stucked + * on long cycles. + */ + kernel_fpu_end(); + schedule(); + kernel_fpu_begin(); + } + sched_helper->free_sched_arg(msg); } + test_conn_release_all(sg); test_sg_release_all(); } diff --git a/tempesta_fw/t/unit/sched_helper.h b/tempesta_fw/t/unit/sched_helper.h index fa49c5dea3..9f2546e14f 100644 --- a/tempesta_fw/t/unit/sched_helper.h +++ b/tempesta_fw/t/unit/sched_helper.h @@ -47,8 +47,12 @@ struct TestSchedHelper { void (*free_sched_arg)(TfwMsg *); }; -void test_sched_generic_empty_sg(struct TestSchedHelper *sched_helper); -void test_sched_generic_one_srv_zero_conn(struct TestSchedHelper *sched_helper); -void test_sched_generic_max_srv_zero_conn(struct TestSchedHelper *sched_helper); +void test_sched_sg_empty_sg(struct TestSchedHelper *sched_helper); +void test_sched_sg_one_srv_zero_conn(struct TestSchedHelper *sched_helper); +void test_sched_sg_max_srv_zero_conn(struct TestSchedHelper *sched_helper); + +void test_sched_srv_one_srv_zero_conn(struct TestSchedHelper *sched_helper); +void test_sched_srv_max_srv_zero_conn(struct TestSchedHelper *sched_helper); +void test_sched_srv_offline_srv(struct TestSchedHelper *sched_helper); #endif /* __TFW_SCHED_HELPER_H__ */ diff --git a/tempesta_fw/t/unit/test_sched_hash.c b/tempesta_fw/t/unit/test_sched_hash.c index fd40669c46..5c6c036821 100644 --- a/tempesta_fw/t/unit/test_sched_hash.c +++ b/tempesta_fw/t/unit/test_sched_hash.c @@ -84,22 +84,15 @@ sched_hash_get_arg(size_t conn_type) TEST(tfw_sched_hash, sg_empty) { - test_sched_generic_empty_sg(&sched_helper_hash); + test_sched_sg_empty_sg(&sched_helper_hash); } -TEST(tfw_sched_hash, one_srv_in_sg_and_zero_conn) +TEST(tfw_sched_hash, sched_sg_one_srv_zero_conn) { - test_sched_generic_one_srv_zero_conn(&sched_helper_hash); + test_sched_sg_one_srv_zero_conn(&sched_helper_hash); } -/* - * This unit test is implementation aware and checks more than just interface. - * Note, that it is very similar to other tests (one_srv_in_sg_and_max_conn and - * max_srv_in_sg_and_max_conn) for round-robin and hash schedullers. So if test - * structure is changed, other mentioned in above tests should be also be - * updated - */ -TEST(tfw_sched_hash, one_srv_in_sg_and_max_conn) +TEST(tfw_sched_hash, sched_sg_one_srv_max_conn) { size_t i, j; @@ -113,40 +106,43 @@ TEST(tfw_sched_hash, one_srv_in_sg_and_max_conn) /* Check that every request is scheduled to the same connection. */ for (i = 0; i < sched_helper_hash.conn_types; ++i) { - TfwSrvConn *expect_conn = NULL; + TfwMsg *msg = sched_helper_hash.get_sched_arg(i); + TfwSrvConn *exp_conn = NULL; for (j = 0; j < TFW_SRV_MAX_CONN; ++j) { - TfwMsg *msg = sched_helper_hash.get_sched_arg(i); - TfwSrvConn *srv_conn = sg->sched->sched_srv(msg, sg); + TfwSrvConn *srv_conn = + sg->sched->sched_sg_conn(msg, sg); EXPECT_NOT_NULL(srv_conn); + if (!srv_conn) + goto err; - if (!expect_conn) - expect_conn = srv_conn; + if (!exp_conn) + exp_conn = srv_conn; else - EXPECT_EQ(srv_conn, expect_conn); + EXPECT_EQ(srv_conn, exp_conn); tfw_srv_conn_put(srv_conn); - sched_helper_hash.free_sched_arg(msg); + /* + * Don't let wachtdog suppose that we have stucked + * on long cycles. + */ + kernel_fpu_end(); + schedule(); + kernel_fpu_begin(); } + sched_helper_hash.free_sched_arg(msg); } - +err: test_conn_release_all(sg); test_sg_release_all(); } -TEST(tfw_sched_hash, max_srv_in_sg_and_zero_conn) +TEST(tfw_sched_hash, sched_sg_max_srv_zero_conn) { - test_sched_generic_max_srv_zero_conn(&sched_helper_hash); + test_sched_sg_max_srv_zero_conn(&sched_helper_hash); } -/* - * This unit test is implementation aware and checks more than just interface. - * Note, that it is very similar to other tests (one_srv_in_sg_and_max_conn and - * max_srv_in_sg_and_max_conn) for round-robin and hash schedullers. So if test - * structure is changed, other mentioned in above tests should be also be - * updated - */ -TEST(tfw_sched_hash, max_srv_in_sg_and_max_conn) +TEST(tfw_sched_hash, sched_sg_max_srv_max_conn) { size_t i, j; @@ -163,27 +159,155 @@ TEST(tfw_sched_hash, max_srv_in_sg_and_max_conn) /* Check that every request is scheduled to the same connection. */ for (i = 0; i < sched_helper_hash.conn_types; ++i) { - TfwSrvConn *expect_conn = NULL; + TfwMsg *msg = sched_helper_hash.get_sched_arg(i); + TfwSrvConn *exp_conn = NULL; for (j = 0; j < TFW_SG_MAX_SRV * TFW_SRV_MAX_CONN; ++j) { - TfwMsg *msg = sched_helper_hash.get_sched_arg(i); - TfwSrvConn *srv_conn = sg->sched->sched_srv(msg, sg); + TfwSrvConn *srv_conn = + sg->sched->sched_sg_conn(msg, sg); + EXPECT_NOT_NULL(srv_conn); + if (!srv_conn) + goto err; + + if (!exp_conn) + exp_conn = srv_conn; + else + EXPECT_EQ(srv_conn, exp_conn); + + tfw_srv_conn_put(srv_conn); + /* + * Don't let wachtdog suppose that we have stucked + * on long cycles. + */ + kernel_fpu_end(); + schedule(); + kernel_fpu_begin(); + } + sched_helper_hash.free_sched_arg(msg); + } +err: + test_conn_release_all(sg); + test_sg_release_all(); +} + +TEST(tfw_sched_hash, sched_srv_one_srv_zero_conn) +{ + test_sched_srv_one_srv_zero_conn(&sched_helper_hash); +} + +TEST(tfw_sched_hash, sched_srv_one_srv_max_conn) +{ + size_t i, j; + + TfwSrvGroup *sg = test_create_sg("test", sched_helper_hash.sched); + TfwServer *srv = test_create_srv("127.0.0.1", sg); + + for (i = 0; i < TFW_SRV_MAX_CONN; ++i) { + TfwSrvConn *srv_conn = test_create_conn((TfwPeer *)srv); + sg->sched->add_conn(sg, srv, srv_conn); + } + + /* Check that every request is scheduled to the same connection. */ + for (i = 0; i < sched_helper_hash.conn_types; ++i) { + TfwMsg *msg = sched_helper_hash.get_sched_arg(i); + TfwSrvConn *exp_conn = NULL; + + for (j = 0; j < TFW_SRV_MAX_CONN; ++j) { + TfwSrvConn *srv_conn = + sg->sched->sched_srv_conn(msg, srv); + EXPECT_NOT_NULL(srv_conn); + if (!srv_conn) + goto err; + EXPECT_EQ((TfwServer *)srv_conn->peer, srv); - if (!expect_conn) - expect_conn = srv_conn; + if (!exp_conn) + exp_conn = srv_conn; else - EXPECT_EQ(srv_conn, expect_conn); + EXPECT_EQ(srv_conn, exp_conn); tfw_srv_conn_put(srv_conn); - sched_helper_hash.free_sched_arg(msg); + /* + * Don't let wachtdog suppose that we have stucked + * on long cycles. + */ + kernel_fpu_end(); + schedule(); + kernel_fpu_begin(); } + sched_helper_hash.free_sched_arg(msg); } +err: + test_conn_release_all(sg); + test_sg_release_all(); +} + +TEST(tfw_sched_hash, sched_srv_max_srv_zero_conn) +{ + test_sched_srv_max_srv_zero_conn(&sched_helper_hash); +} + +TEST(tfw_sched_hash, sched_srv_max_srv_max_conn) +{ + size_t i, j; + + TfwSrvGroup *sg = test_create_sg("test", sched_helper_hash.sched); + + for (i = 0; i < TFW_SG_MAX_SRV; ++i) { + TfwServer *srv = test_create_srv("127.0.0.1", sg); + for (j = 0; j < TFW_SRV_MAX_CONN; ++j) { + TfwSrvConn *srv_conn = + test_create_conn((TfwPeer *)srv); + sg->sched->add_conn(sg, srv, srv_conn); + } + } + + /* Check that every request is scheduled to the same connection. */ + for (i = 0; i < sched_helper_hash.conn_types; ++i) { + TfwMsg *msg = sched_helper_hash.get_sched_arg(i); + TfwServer *srv; + + list_for_each_entry(srv, &sg->srv_list, list) { + TfwSrvConn *exp_conn = NULL; + + for (j = 0; j < TFW_SG_MAX_SRV * TFW_SRV_MAX_CONN; ++j) { + TfwSrvConn *srv_conn = + sg->sched->sched_srv_conn(msg, srv); + + EXPECT_NOT_NULL(srv_conn); + if (!srv_conn) + goto err; + EXPECT_EQ((TfwServer *)srv_conn->peer, srv); + + if (!exp_conn) + exp_conn = srv_conn; + else + EXPECT_EQ(srv_conn, exp_conn); + + tfw_srv_conn_put(srv_conn); + + /* + * Don't let wachtdog suppose that we have + * stucked on long cycles. + */ + kernel_fpu_end(); + schedule(); + kernel_fpu_begin(); + } + } + sched_helper_hash.free_sched_arg(msg); + } +err: test_conn_release_all(sg); test_sg_release_all(); } +TEST(tfw_sched_hash, sched_srv_offline_srv) +{ + test_sched_srv_offline_srv(&sched_helper_hash); +} + TEST_SUITE(sched_hash) { kernel_fpu_end(); @@ -193,9 +317,29 @@ TEST_SUITE(sched_hash) kernel_fpu_begin(); + /* + * Schedulers have the same interface so some test cases can use generic + * implementations. Some test cases still have to know how scheduler + * work at low level. Please, keep same structure for implementation + * aware test cases across all schedulers. + * + * Implementation aware cases: + * sched_sg_one_srv_max_conn + * sched_sg_max_srv_max_conn + * sched_srv_one_srv_max_conn + * sched_srv_max_srv_max_conn + */ + TEST_RUN(tfw_sched_hash, sg_empty); - TEST_RUN(tfw_sched_hash, one_srv_in_sg_and_zero_conn); - TEST_RUN(tfw_sched_hash, one_srv_in_sg_and_max_conn); - TEST_RUN(tfw_sched_hash, max_srv_in_sg_and_zero_conn); - TEST_RUN(tfw_sched_hash, max_srv_in_sg_and_max_conn); + + TEST_RUN(tfw_sched_hash, sched_sg_one_srv_zero_conn); + TEST_RUN(tfw_sched_hash, sched_sg_one_srv_max_conn); + TEST_RUN(tfw_sched_hash, sched_sg_max_srv_zero_conn); + TEST_RUN(tfw_sched_hash, sched_sg_max_srv_max_conn); + + TEST_RUN(tfw_sched_hash, sched_srv_one_srv_zero_conn); + TEST_RUN(tfw_sched_hash, sched_srv_one_srv_max_conn); + TEST_RUN(tfw_sched_hash, sched_srv_max_srv_zero_conn); + TEST_RUN(tfw_sched_hash, sched_srv_max_srv_max_conn); + TEST_RUN(tfw_sched_hash, sched_srv_offline_srv); } diff --git a/tempesta_fw/t/unit/test_sched_http.c b/tempesta_fw/t/unit/test_sched_http.c index 426a051b5a..6ec104fe4e 100644 --- a/tempesta_fw/t/unit/test_sched_http.c +++ b/tempesta_fw/t/unit/test_sched_http.c @@ -301,7 +301,7 @@ TestCase test_cases[] = { }, }; -size_t test_cases_size = sizeof(test_cases) / sizeof(test_cases[0]); +size_t test_cases_size = ARRAY_SIZE(test_cases); TEST(tfw_sched_http, one_rule) { diff --git a/tempesta_fw/t/unit/test_sched_rr.c b/tempesta_fw/t/unit/test_sched_rr.c index 2202f21e5f..f2585046c9 100644 --- a/tempesta_fw/t/unit/test_sched_rr.c +++ b/tempesta_fw/t/unit/test_sched_rr.c @@ -64,22 +64,15 @@ static struct TestSchedHelper sched_helper_rr = { TEST(tfw_sched_rr, sg_empty) { - test_sched_generic_empty_sg(&sched_helper_rr); + test_sched_sg_empty_sg(&sched_helper_rr); } -TEST(tfw_sched_rr, one_srv_in_sg_and_zero_conn) +TEST(tfw_sched_rr, sched_sg_one_srv_zero_conn) { - test_sched_generic_one_srv_zero_conn(&sched_helper_rr); + test_sched_sg_one_srv_zero_conn(&sched_helper_rr); } -/* - * This unit test is implementation aware and checks more than just interface. - * Note, that it is very similar to other tests (one_srv_in_sg_and_max_conn and - * max_srv_in_sg_and_max_conn) for round-robin and hash schedullers. So if test - * structure is changed, other mentioned in above tests should be also be - * updated - */ -TEST(tfw_sched_rr, one_srv_in_sg_and_max_conn) +TEST(tfw_sched_rr, sched_sg_one_srv_max_conn) { size_t i, j; long long conn_acc = 0, conn_acc_check = 0; @@ -98,38 +91,41 @@ TEST(tfw_sched_rr, one_srv_in_sg_and_max_conn) * every connection will be scheduled only once */ for (i = 0; i < sched_helper_rr.conn_types; ++i) { + TfwMsg *msg = sched_helper_rr.get_sched_arg(i); conn_acc_check = 0; for (j = 0; j < TFW_SRV_MAX_CONN; ++j) { - TfwMsg *msg = sched_helper_rr.get_sched_arg(i); - TfwSrvConn *srv_conn = sg->sched->sched_srv(msg, sg); + TfwSrvConn *srv_conn = + sg->sched->sched_sg_conn(msg, sg); EXPECT_NOT_NULL(srv_conn); + if (!srv_conn) + goto err; conn_acc_check ^= (long long)srv_conn; tfw_srv_conn_put(srv_conn); - sched_helper_rr.free_sched_arg(msg); + /* + * Don't let wachtdog suppose that we have stucked + * on long cycles. + */ + kernel_fpu_end(); + schedule(); + kernel_fpu_begin(); } EXPECT_EQ(conn_acc, conn_acc_check); + sched_helper_rr.free_sched_arg(msg); } - +err: test_conn_release_all(sg); test_sg_release_all(); } -TEST(tfw_sched_rr, max_srv_in_sg_and_zero_conn) +TEST(tfw_sched_rr, sched_sg_max_srv_zero_conn) { - test_sched_generic_max_srv_zero_conn(&sched_helper_rr); + test_sched_sg_max_srv_zero_conn(&sched_helper_rr); } -/* - * This unit test is implementation aware and checks more than just interface. - * Note, that it is very similar to other tests (one_srv_in_sg_and_max_conn and - * max_srv_in_sg_and_max_conn) for round-robin and hash schedullers. So if test - * structure is changed, other mentioned in above tests should be also be - * updated - */ -TEST(tfw_sched_rr, max_srv_in_sg_and_max_conn) +TEST(tfw_sched_rr, sched_sg_max_srv_max_conn) { size_t i, j; long long conn_acc = 0, conn_acc_check = 0; @@ -151,25 +147,160 @@ TEST(tfw_sched_rr, max_srv_in_sg_and_max_conn) * every connection will be scheduled only once */ for (i = 0; i < sched_helper_rr.conn_types; ++i) { + TfwMsg *msg = sched_helper_rr.get_sched_arg(i); conn_acc_check = 0; for (j = 0; j < TFW_SG_MAX_SRV * TFW_SRV_MAX_CONN; ++j) { - TfwMsg *msg = sched_helper_rr.get_sched_arg(i); - TfwSrvConn *srv_conn = sg->sched->sched_srv(msg, sg); + TfwSrvConn *srv_conn = + sg->sched->sched_sg_conn(msg, sg); EXPECT_NOT_NULL(srv_conn); + if (!srv_conn) + goto err; conn_acc_check ^= (long long)srv_conn; tfw_srv_conn_put(srv_conn); - sched_helper_rr.free_sched_arg(msg); } EXPECT_EQ(conn_acc, conn_acc_check); + sched_helper_rr.free_sched_arg(msg); + } +err: + test_conn_release_all(sg); + test_sg_release_all(); +} + +TEST(tfw_sched_rr, sched_srv_one_srv_zero_conn) +{ + test_sched_srv_one_srv_zero_conn(&sched_helper_rr); +} + +TEST(tfw_sched_rr, sched_srv_one_srv_max_conn) +{ + size_t i, j; + long long conn_acc = 0, conn_acc_check = 0; + + TfwSrvGroup *sg = test_create_sg("test", sched_helper_rr.sched); + TfwServer *srv = test_create_srv("127.0.0.1", sg); + + for (i = 0; i < TFW_SRV_MAX_CONN; ++i) { + TfwSrvConn *srv_conn = test_create_conn((TfwPeer *)srv); + sg->sched->add_conn(sg, srv, srv_conn); + conn_acc ^= (long long)srv_conn; } + /* + * Check that connections is scheduled in the fair way: + * every connection will be scheduled only once + */ + for (i = 0; i < sched_helper_rr.conn_types; ++i) { + TfwMsg *msg = sched_helper_rr.get_sched_arg(i); + conn_acc_check = 0; + + for (j = 0; j < TFW_SRV_MAX_CONN; ++j) { + TfwSrvConn *srv_conn = + sg->sched->sched_srv_conn(msg, srv); + EXPECT_NOT_NULL(srv_conn); + if (!srv_conn) + goto err; + EXPECT_EQ((TfwServer *)srv_conn->peer, srv); + + conn_acc_check ^= (long long)srv_conn; + tfw_srv_conn_put(srv_conn); + + /* + * Don't let wachtdog wuppose that we have stucked + * on long cycles. + */ + kernel_fpu_end(); + schedule(); + kernel_fpu_begin(); + } + + EXPECT_EQ(conn_acc, conn_acc_check); + sched_helper_rr.free_sched_arg(msg); + } +err: test_conn_release_all(sg); test_sg_release_all(); } +TEST(tfw_sched_rr, sched_srv_max_srv_zero_conn) +{ + test_sched_srv_max_srv_zero_conn(&sched_helper_rr); +} + +TEST(tfw_sched_rr, sched_srv_max_srv_max_conn) +{ + size_t i, j; + long long conn_acc_check = 0; + struct { + TfwServer *srv; + long long conn_acc; + } srv_acc[TFW_SG_MAX_SRV] = { 0 }; + + TfwSrvGroup *sg = test_create_sg("test", sched_helper_rr.sched); + + for (i = 0; i < TFW_SG_MAX_SRV; ++i) { + TfwServer *srv = test_create_srv("127.0.0.1", sg); + srv_acc[i].srv = srv; + + for (j = 0; j < TFW_SRV_MAX_CONN; ++j) { + TfwSrvConn *srv_conn = test_create_conn((TfwPeer *)srv); + sg->sched->add_conn(sg, srv, srv_conn); + srv_acc[i].conn_acc ^= (long long)srv_conn; + } + } + + /* + * Check that connections is scheduled in the fair way: + * every connection will be scheduled only once + */ + for (i = 0; i < sched_helper_rr.conn_types; ++i) { + TfwMsg *msg = sched_helper_rr.get_sched_arg(i); + TfwServer *srv; + + list_for_each_entry(srv, &sg->srv_list, list) { + size_t k = 0; + conn_acc_check = 0; + + for (j = 0; j < TFW_SRV_MAX_CONN; ++j) { + TfwSrvConn *srv_conn = + sg->sched->sched_srv_conn(msg, srv); + EXPECT_NOT_NULL(srv_conn); + if (!srv_conn) + goto err; + EXPECT_EQ((TfwServer *)srv_conn->peer, srv); + + conn_acc_check ^= (long long)srv_conn; + tfw_srv_conn_put(srv_conn); + + /* + * Don't let wachtdog wuppose that we have + * stucked on long cycles. + */ + kernel_fpu_end(); + schedule(); + kernel_fpu_begin(); + } + + for (k = 0; k < TFW_SG_MAX_SRV; ++k) { + if (srv_acc[k].srv == srv) + EXPECT_EQ(srv_acc[k].conn_acc, + conn_acc_check); + } + } + sched_helper_rr.free_sched_arg(msg); + } +err: + test_conn_release_all(sg); + test_sg_release_all(); +} + +TEST(tfw_sched_rr, sched_srv_offline_srv) +{ + test_sched_srv_offline_srv(&sched_helper_rr); +} + TEST_SUITE(sched_rr) { kernel_fpu_end(); @@ -179,9 +310,29 @@ TEST_SUITE(sched_rr) kernel_fpu_begin(); + /* + * Schedulers have the same interface so some test cases can use generic + * implementations. Some test cases still have to know how scheduler + * work at low level. Please, keep same structure for implementation + * aware test cases across all schedulers. + * + * Implementation aware cases: + * sched_sg_one_srv_max_conn + * sched_sg_max_srv_max_conn + * sched_srv_one_srv_max_conn + * sched_srv_max_srv_max_conn + */ + TEST_RUN(tfw_sched_rr, sg_empty); - TEST_RUN(tfw_sched_rr, one_srv_in_sg_and_zero_conn); - TEST_RUN(tfw_sched_rr, one_srv_in_sg_and_max_conn); - TEST_RUN(tfw_sched_rr, max_srv_in_sg_and_zero_conn); - TEST_RUN(tfw_sched_rr, max_srv_in_sg_and_max_conn); + + TEST_RUN(tfw_sched_rr, sched_sg_one_srv_zero_conn); + TEST_RUN(tfw_sched_rr, sched_sg_one_srv_max_conn); + TEST_RUN(tfw_sched_rr, sched_sg_max_srv_zero_conn); + TEST_RUN(tfw_sched_rr, sched_sg_max_srv_max_conn); + + TEST_RUN(tfw_sched_rr, sched_srv_one_srv_zero_conn); + TEST_RUN(tfw_sched_rr, sched_srv_one_srv_max_conn); + TEST_RUN(tfw_sched_rr, sched_srv_max_srv_zero_conn); + TEST_RUN(tfw_sched_rr, sched_srv_max_srv_max_conn); + TEST_RUN(tfw_sched_rr, sched_srv_offline_srv); }