diff --git a/tempesta_fw/cache.c b/tempesta_fw/cache.c index 2183245ee9..3293ef236e 100644 --- a/tempesta_fw/cache.c +++ b/tempesta_fw/cache.c @@ -34,6 +34,7 @@ #include "vhost.h" #include "cache.h" #include "http_msg.h" +#include "http_sess.h" #include "procfs.h" #include "sync_socket.h" #include "work_queue.h" @@ -67,10 +68,11 @@ static const TfwStr tfw_cache_raw_headers_304[] = { /* * @trec - Database record descriptor; * @key_len - length of key (URI + Host header); - * @status_len - length of response status line; + * @status_len - length of response status code; + * @rph_len - length of response reason phrase; * @hdr_num - number of headers; * @hdr_len - length of whole headers data; - * @hdr_len_304 - length of headers data used to build 304 response; + * @body_len - length of the response body; * @method - request method, part of the key; * @flags - various cache entry flags; * @age - the value of response Age: header field; @@ -80,9 +82,9 @@ static const TfwStr tfw_cache_raw_headers_304[] = { * @lifetime - the cache entry's current lifetime; * @last_modified - the value of response Last-Modified: header field; * @key - the cache entry key (URI + Host header); - * @status - pointer to status line (with trailing CRLFs); - * @hdrs - pointer to list of HTTP headers (with trailing CRLFs); - * @body - pointer to response body (with a prepending CRLF); + * @status - pointer to status line; + * @hdrs - pointer to list of HTTP headers; + * @body - pointer to response body; * @hdrs_304 - pointers to headers used to build 304 response; * @version - HTTP version of the response; * @resp_status - Http status of the cached response. @@ -94,9 +96,10 @@ typedef struct { #define ce_body key_len unsigned int key_len; unsigned int status_len; + unsigned int rph_len; unsigned int hdr_num; unsigned int hdr_len; - unsigned int hdr_len_304; + unsigned int body_len; unsigned int method: 4; unsigned int flags: 28; time_t age; @@ -169,6 +172,8 @@ typedef struct { static CaNode c_nodes[MAX_NUMNODES]; +typedef int tfw_cache_write_actor_t(TDB *, TdbVRec **, TfwHttpResp *, char **, + size_t, TfwDecodeCacheIter *); /* * TODO the thread doesn't do anything for now, however, kthread_stop() crashes * on restarts, so comment to logic out. @@ -178,25 +183,27 @@ static struct task_struct *cache_mgr_thr; #endif static DEFINE_PER_CPU(TfwWorkTasklet, cache_wq); +#define RESP_BUF_LEN 128 + +static DEFINE_PER_CPU(char[RESP_BUF_LEN], g_c_buf); + static TfwStr g_crlf = { .data = S_CRLF, .len = SLEN(S_CRLF) }; /* Iterate over request URI and Host header to process request key. */ -#define TFW_CACHE_REQ_KEYITER(c, req, u_end, h_start, h_end) \ - if (TFW_STR_PLAIN(&req->uri_path)) { \ - c = &req->uri_path; \ - u_end = &req->uri_path + 1; \ +#define TFW_CACHE_REQ_KEYITER(c, uri_path, host, u_end, h_start, h_end) \ + if (TFW_STR_PLAIN(uri_path)) { \ + c = uri_path; \ + u_end = (uri_path) + 1; \ } else { \ - c = req->uri_path.chunks; \ - u_end = req->uri_path.chunks \ - + req->uri_path.nchunks; \ + c = (uri_path)->chunks; \ + u_end = (uri_path)->chunks + (uri_path)->nchunks; \ } \ - if (TFW_STR_PLAIN(&req->h_tbl->tbl[TFW_HTTP_HDR_HOST])) { \ - h_start = req->h_tbl->tbl + TFW_HTTP_HDR_HOST; \ - h_end = req->h_tbl->tbl + TFW_HTTP_HDR_HOST + 1; \ + if (!TFW_STR_EMPTY(host)) { \ + BUG_ON(TFW_STR_PLAIN(host)); \ + h_start = (host)->chunks; \ + h_end = (host)->chunks + (host)->nchunks; \ } else { \ - h_start = req->h_tbl->tbl[TFW_HTTP_HDR_HOST].chunks; \ - h_end = req->h_tbl->tbl[TFW_HTTP_HDR_HOST].chunks \ - + req->h_tbl->tbl[TFW_HTTP_HDR_HOST].nchunks; \ + h_start = h_end = u_end; \ } \ for ( ; c != h_end; ++c, c = (c == u_end) ? h_start : c) @@ -531,29 +538,38 @@ tfw_cache_cond_none_match(TfwHttpReq *req, TfwCacheEntry *ce) } /** - * Add a new data chunk size of @len starting from @data to HTTP response @resp. - * Properly set up @hdr if not NULL. + * Add a new header with size of @len starting from @data to HTTP response @resp, + * expanding the @resp with new skb/frags if needed. */ static int -tfw_cache_write_field(TDB *db, TdbVRec **trec, TfwHttpResp *resp, - TfwMsgIter *it, char **data, size_t len, TfwStr *hdr) +tfw_cache_h2_write(TDB *db, TdbVRec **trec, TfwHttpResp *resp, char **data, + size_t len, TfwDecodeCacheIter *dc_iter) { - int r, copied = 0; - TdbVRec *tr = *trec; TfwStr c = { 0 }; + TdbVRec *tr = *trec; + TfwHttpTransIter *mit = &resp->mit; + TfwMsgIter *it = &mit->iter; + int r = 0, copied = 0; while (1) { c.data = *data; - c.len = min(tr->data + tr->len - *data, - (long)(len - copied)); - r = hdr - ? tfw_http_msg_add_data(it, (TfwHttpMsg *)resp, hdr, &c) - : tfw_msg_write(it, &c); - if (r) - return r; + c.len = min(tr->data + tr->len - *data, (long)(len - copied)); + if (!dc_iter->skip) { + r = tfw_http_msg_expand_data(it, &resp->msg.skb_head, + &c, &mit->start_off); + if (unlikely(r)) + break; + + dc_iter->acc_len += c.len; + } copied += c.len; *data += c.len; + + T_DBG3("%s: len='%zu', copied='%d', dc_iter->acc_len='%lu'," + " dc_iter->skip='%d'\n", __func__, len, copied, + dc_iter->acc_len, dc_iter->skip); + if (copied == len) break; @@ -562,56 +578,152 @@ tfw_cache_write_field(TDB *db, TdbVRec **trec, TfwHttpResp *resp, *data = tr->data; } - /* Every non-empty header contains CRLF at the end. We need to translate - * it to { str, eolen } presentation. */ - if (hdr && hdr->len) - tfw_str_fixup_eol(hdr, SLEN(S_CRLF)); + return r; +} + +/** + * Same as @tfw_cache_h2_write(), but also decode the header from HTTP/2 format + * before writing it into the response (used e.g. for HTTP/1.1-response creation + * from cache). + */ +static int +tfw_cache_h2_decode_write(TDB *db, TdbVRec **trec, TfwHttpResp *resp, + char **data, size_t len, TfwDecodeCacheIter *dc_iter) +{ + unsigned long m_len; + TdbVRec *tr = *trec; + int r = 0, acc = 0; + TfwHPack hp = {}; + + while (1) { + m_len = min(tr->data + tr->len - *data, (long)(len - acc)); + if (!dc_iter->skip) { + r = tfw_hpack_cache_decode_expand(&hp, resp, *data, + m_len, dc_iter); + if (unlikely(r)) + break; + } + + acc += m_len; + *data += m_len; + if (acc == len) { + bzero_fast(dc_iter->__off, + sizeof(*dc_iter) + - offsetof(TfwDecodeCacheIter, __off)); + break; + } + + tr = *trec = tdb_next_rec_chunk(db, tr); + BUG_ON(!tr); + *data = tr->data; + } + + return r; +} + +static int +tfw_cache_set_status(TDB *db, TfwCacheEntry *ce, TfwHttpResp *resp, + TdbVRec **trec, char **p, unsigned long *acc_len) +{ + int r; + TfwMsgIter *it = &resp->mit.iter; + struct sk_buff **skb_head = &resp->msg.skb_head; + bool h2_mode = TFW_MSG_H2(resp->req); + TfwDecodeCacheIter dc_iter = {}; + + if (h2_mode) + resp->mit.start_off = FRAME_HEADER_SIZE; + else + dc_iter.skip = true; + + + r = tfw_cache_h2_write(db, trec, resp, p, ce->status_len, &dc_iter); + if (unlikely(r)) + return r; + + if (!h2_mode) { + char buf[H2_STAT_VAL_LEN]; + TfwStr s_line = { + .chunks = (TfwStr []){ + { .data = S_0, .len = SLEN(S_0) }, + { .data = buf, .len = H2_STAT_VAL_LEN} + }, + .len = SLEN(S_0) + H2_STAT_VAL_LEN, + .nchunks = 2 + }; + + if (!tfw_ultoa(ce->resp_status, __TFW_STR_CH(&s_line, 1)->data, + H2_STAT_VAL_LEN)) + return -E2BIG; + + r = tfw_http_msg_expand_data(it, skb_head, &s_line, NULL); + if (unlikely(r)) + return r; + + *acc_len += s_line.len; + } + + dc_iter.skip = h2_mode ? true : false; + + r = tfw_cache_h2_write(db, trec, resp, p, ce->rph_len, &dc_iter); + if (unlikely(r)) + return r; + + *acc_len += dc_iter.acc_len; + + if (!h2_mode) { + r = tfw_http_msg_expand_data(it, skb_head, &g_crlf, NULL); + if (unlikely(r)) + return r; + + *acc_len += g_crlf.len; + } return 0; } /** * Write HTTP header to skb data. - * The headers are likely to be adjusted, so copy them. */ static int -tfw_cache_build_resp_hdr(TDB *db, TfwHttpResp *resp, TfwStr *hdr, - TdbVRec **trec, TfwMsgIter *it, char **p) +tfw_cache_build_resp_hdr(TDB *db, TfwHttpResp *resp, TfwHdrMods *hmods, + TdbVRec **trec, char **p, unsigned long *acc_len) { - int r, d, dn; - TfwStr *dups; + tfw_cache_write_actor_t *write_actor; TfwCStr *s = (TfwCStr *)*p; + TfwHttpReq *req = resp->req; + TfwDecodeCacheIter dc_iter = { .h_mods = hmods }; + int d, dn, r = 0; + + BUG_ON(!req); + + write_actor = !TFW_MSG_H2(req) || dc_iter.h_mods + ? tfw_cache_h2_decode_write + : tfw_cache_h2_write; *p += TFW_CSTR_HDRLEN; BUG_ON(*p > (*trec)->data + (*trec)->len); - if (likely(!(s->flags & TFW_STR_DUPLICATE))) - return tfw_cache_write_field(db, trec, resp, it, p, s->len, - hdr); + if (likely(!(s->flags & TFW_STR_DUPLICATE))) { + r = write_actor(db, trec, resp, p, s->len, &dc_iter); + if (likely(!r)) + *acc_len += dc_iter.acc_len; + return r; + } /* Process duplicated headers. */ dn = s->len; - dups = tfw_pool_alloc(resp->pool, dn * sizeof(TfwStr)); - if (!dups) - return -ENOMEM; - for (d = 0; d < dn; ++d) { s = (TfwCStr *)*p; BUG_ON(s->flags); - TFW_STR_INIT(&dups[d]); *p += TFW_CSTR_HDRLEN; - if ((r = tfw_cache_write_field(db, trec, resp, it, p, s->len, - &dups[d]))) - return r; - } - - if (hdr) { - hdr->chunks = dups; - hdr->nchunks = dn; - hdr->flags |= TFW_STR_DUPLICATE; + r = write_actor(db, trec, resp, p, s->len, &dc_iter); + if (unlikely(r)) + break; + *acc_len += dc_iter.acc_len;; } - return 0; + return r; } /** @@ -620,37 +732,49 @@ tfw_cache_build_resp_hdr(TDB *db, TfwHttpResp *resp, TfwStr *hdr, * Last-Modified might be used if the response does not have an ETag field. * * The 304 response should be as short as possible, we don't need to add - * extra headers with tfw_http_adjust_resp(). Use quicker tfw_msg_write() - * instead of tfw_http_msg_add_data() used to build full response. + * extra headers. */ static void tfw_cache_send_304(TfwHttpReq *req, TfwCacheEntry *ce) { - TfwHttpResp *resp; - TfwMsgIter it; - int i; char *p; + int r, i; + TfwMsgIter *it; + TfwHttpResp *resp; + TfwFrameHdr frame_hdr; + struct sk_buff **skb_head; + unsigned char buf[FRAME_HEADER_SIZE]; + unsigned int stream_id = 0; + unsigned long h_len = 0; TdbVRec *trec = &ce->trec; TDB *db = node_db(); WARN_ON_ONCE(!list_empty(&req->fwd_list)); WARN_ON_ONCE(!list_empty(&req->nip_list)); - if (TFW_MSG_H2(req)) { - /* - * TODO #309: add separate flow for HTTP/2 response preparing - * and sending (HPACK index, encode in HTTP/2 format, add frame - * headers and send via @tfw_h2_resp_fwd()). - */ - return; - } - if (!(resp = tfw_http_msg_alloc_resp_light(req))) goto err_create; - if (tfw_http_prep_304((TfwHttpMsg *)resp, req, &it, - ce->hdr_len_304)) - goto err_setup; + it = &resp->mit.iter; + skb_head = &resp->msg.skb_head; + + if (!TFW_MSG_H2(req)) { + r = tfw_http_prep_304(req, skb_head, it); + if (unlikely(r)) + goto err_setup; + } else { + stream_id = tfw_h2_stream_id_close(req, HTTP2_HEADERS, + HTTP2_F_END_STREAM); + if (unlikely(!stream_id)) + goto err_setup; + + resp->mit.start_off = FRAME_HEADER_SIZE; + + r = tfw_h2_resp_status_write(resp, 304, TFW_H2_TRANS_EXPAND, + true); + if (unlikely(r)) + goto err_setup; + } /* Put 304 headers */ for (i = 0; i < ARRAY_SIZE(ce->hdrs_304); ++i) { @@ -662,14 +786,31 @@ tfw_cache_send_304(TfwHttpReq *req, TfwCacheEntry *ce) trec = tdb_next_rec_chunk(db, trec); BUG_ON(!trec); - if (tfw_cache_build_resp_hdr(db, resp, NULL, &trec, &it, &p)) + if (tfw_cache_build_resp_hdr(db, resp, NULL, &trec, &p, &h_len)) goto err_setup; } - if (tfw_msg_write(&it, &g_crlf)) + if (!TFW_MSG_H2(req)) { + if (tfw_http_msg_expand_data(it, skb_head, &g_crlf, NULL)) + goto err_setup; + + tfw_http_resp_fwd(resp); + + return; + } + + if (h_len > FRAME_MAX_LENGTH) goto err_setup; - tfw_http_resp_fwd(resp); + frame_hdr.stream_id = stream_id; + frame_hdr.length = h_len; + frame_hdr.type = HTTP2_HEADERS; + frame_hdr.flags = HTTP2_F_END_HEADERS | HTTP2_F_END_STREAM; + + tfw_h2_pack_frame_header(buf, &frame_hdr); + memcpy_fast((*skb_head)->data, buf, sizeof(buf)); + + tfw_h2_resp_fwd(resp); return; err_setup: @@ -749,15 +890,24 @@ tfw_cache_entry_key_eq(TDB *db, TfwHttpReq *req, TfwCacheEntry *ce) int n, c_off = 0, t_off; TdbVRec *trec = &ce->trec; TfwStr *c, *h_start, *u_end, *h_end; + TfwStr host_val, *host = &req->h_tbl->tbl[TFW_HTTP_HDR_HOST]; if ((req->method != TFW_HTTP_METH_PURGE) && (ce->method != req->method)) return false; - if (req->uri_path.len - + req->h_tbl->tbl[TFW_HTTP_HDR_HOST].len != ce->key_len) + + /* + * Get 'host' header value (from HTTP/2 or HTTP/1.1 request) for + * strict comparison. + */ + tfw_http_msg_clnthdr_val(req, host, TFW_HTTP_HDR_HOST, &host_val); + + if (req->uri_path.len + host_val.len != ce->key_len) return false; t_off = CE_BODY_SIZE; - TFW_CACHE_REQ_KEYITER(c, req, u_end, h_start, h_end) { + TFW_CACHE_REQ_KEYITER(c, &req->uri_path, &host_val, u_end, h_start, + h_end) + { if (!trec) return false; this_chunk: @@ -836,20 +986,6 @@ tfw_cache_dbce_put(TfwCacheEntry *ce) tdb_rec_put(ce); } -static void -tfw_cache_str_write_hdr(const TfwStr *str, char *p) -{ - TfwCStr *s = (TfwCStr *)p; - - if (TFW_STR_DUP(str)) { - s->flags = TFW_STR_DUPLICATE; - s->len = str->nchunks; - } else { - s->flags = 0; - s->len = str->len ? str->len + SLEN(S_CRLF) : 0; - } -} - /** * Copies plain TfwStr @src to TdbRec @trec. * @return number of copied bytes (@src length). @@ -915,18 +1051,31 @@ tfw_cache_strcpy_lc(char **p, TdbVRec **trec, TfwStr *src, size_t tot_len) return __tfw_cache_strcpy(p, trec, src, tot_len, tfw_cstrtolower); } +#define CSTR_MOVE_HDR() \ +do { \ + cs = (TfwCStr *)*p; \ + *p += TFW_CSTR_HDRLEN; \ + *tot_len -= TFW_CSTR_HDRLEN; \ + ce->hdr_len += TFW_CSTR_HDRLEN; \ +} while (0) + +#define CSTR_WRITE_HDR(f, l) \ +do { \ + cs->flags = f; \ + cs->len = l; \ +} while (0) + /** - * Copies plain or compound (chunked) TfwStr @src to TdbRec @trec may be - * appending EOL marker at the end. + * Copies plain or compound (chunked) TfwStr @src to TdbRec @trec. * - * @src is copied (possibly with EOL appended) + * @src is copied * @return number of copied bytes on success and negative value otherwise. */ -static long -tfw_cache_strcpy_eol(char **p, TdbVRec **trec, - TfwStr *src, size_t *tot_len, bool eol) +static int +tfw_cache_h2_copy_str(unsigned int *acc_len, char **p, TdbVRec **trec, + TfwStr *src, size_t *tot_len) { - long n, copied = 0; + long n; TfwStr *c, *end; BUG_ON(TFW_STR_DUP(src)); @@ -936,73 +1085,170 @@ tfw_cache_strcpy_eol(char **p, TdbVRec **trec, TFW_STR_FOR_EACH_CHUNK(c, src, end) { if ((n = tfw_cache_strcpy(p, trec, c, *tot_len)) < 0) { - T_ERR("Cache: cannot copy chunk of string\n"); + T_ERR("Cache: cannot copy chunk of HTTP/2 string\n"); return -ENOMEM; } - *tot_len -= n; - copied += n; - } - if (eol) { - if ((n = tfw_cache_strcpy(p, trec, &g_crlf, *tot_len)) < 0) - return -ENOMEM; - BUG_ON(n != SLEN(S_CRLF)); *tot_len -= n; - copied += n; + *acc_len += n; } - return copied; + return 0; +} + +static inline int +tfw_cache_h2_copy_int(unsigned int *acc_len, unsigned long src, + unsigned short max, char **p, TdbVRec **trec, + size_t *tot_len) +{ + int r; + TfwHPackInt hp_int; + TfwStr str = {}; + + write_int(src, max, 0, &hp_int); + + str.data = hp_int.buf; + str.len = hp_int.sz; + + if ((r = tfw_cache_h2_copy_str(acc_len, p, trec, &str, tot_len))) + return r; + + return 0; } /** * Deep HTTP header copy to TdbRec. - * @src is copied in depth first fashion to speed up upcoming scans. + * @hdr is copied in depth first fashion to speed up upcoming scans. * @return number of copied bytes on success and negative value otherwise. */ static long -tfw_cache_copy_hdr(char **p, TdbVRec **trec, TfwStr *src, size_t *tot_len) +tfw_cache_h2_copy_hdr(TfwCacheEntry *ce, char **p, TdbVRec **trec, TfwStr *hdr, + size_t *tot_len) { - long n = sizeof(TfwCStr), copied; - TfwStr *dup, *dup_end; + TfwCStr *cs; + long n = sizeof(TfwCStr); + unsigned short st_index = 0; + bool dupl = TFW_STR_DUP(hdr); + unsigned int init_len = ce->hdr_len; + TfwStr s_nm, s_val, *dup, *dup_end; - if (unlikely(src->len >= TFW_CSTR_MAXLEN)) { - T_WARN("Cache: trying to store too big string %lx\n", - src->len); - return -E2BIG; - } - /* Don't split short strings. */ - if (likely(!TFW_STR_DUP(src)) - && sizeof(TfwCStr) + src->len <= L1_CACHE_BYTES) - n += src->len; + T_DBG3("%s: ce=[%p] p=[%p], trec=[%p], tot_len='%zu'\n", __func__, ce, + *p, *trec, *tot_len); -#define CSTR_WRITE_HDR(str) \ - tfw_cache_str_write_hdr(str, *p); \ - *p += TFW_CSTR_HDRLEN; \ - *tot_len -= TFW_CSTR_HDRLEN; \ - copied = TFW_CSTR_HDRLEN; + if (likely(!dupl)) { + unsigned long h_len; + + TFW_STR_INIT(&s_nm); + TFW_STR_INIT(&s_val); + tfw_http_hdr_split(hdr, &s_nm, &s_val, true); + + st_index = hdr->hpack_idx; + h_len = tfw_h2_hdr_size(s_nm.len, s_val.len, st_index); + + /* Don't split short strings. */ + if (sizeof(TfwCStr) + h_len <= L1_CACHE_BYTES) + n += h_len; + } *p = tdb_entry_get_room(node_db(), trec, *p, n, *tot_len); - if (!*p) { + if (unlikely(!*p)) { T_WARN("Cache: cannot allocate TDB space\n"); return -ENOMEM; } - CSTR_WRITE_HDR(src); + CSTR_MOVE_HDR(); - if (!TFW_STR_DUP(src)) { - if ((n = tfw_cache_strcpy_eol(p, trec, src, tot_len, 1)) < 0) - return n; - return copied + n; + if (TFW_STR_DUP(hdr)) { + CSTR_WRITE_HDR(TFW_STR_DUPLICATE, hdr->nchunks); + CSTR_MOVE_HDR(); } - TFW_STR_FOR_EACH_DUP(dup, src, dup_end) { - CSTR_WRITE_HDR(dup); - if ((n = tfw_cache_strcpy_eol(p, trec, dup, tot_len, 1)) < 0) - return n; - copied += n; + TFW_STR_FOR_EACH_DUP(dup, hdr, dup_end) { + unsigned int prev_len = ce->hdr_len; + + if (dupl) { + TFW_STR_INIT(&s_nm); + TFW_STR_INIT(&s_val); + tfw_http_hdr_split(dup, &s_nm, &s_val, true); + st_index = dup->hpack_idx; + CSTR_MOVE_HDR(); + } + + if (st_index) { + if (tfw_cache_h2_copy_int(&ce->hdr_len, st_index, 0xF, + p, trec, tot_len)) + return -ENOMEM; + } + else { + if (tfw_cache_h2_copy_int(&ce->hdr_len, 0, 0xF, p, trec, + tot_len) + || tfw_cache_h2_copy_int(&ce->hdr_len, s_nm.len, + 0x7f, p, trec, tot_len) + || tfw_cache_h2_copy_str(&ce->hdr_len, p, trec, + &s_nm, tot_len)) + return -ENOMEM; + } + + if (tfw_cache_h2_copy_int(&ce->hdr_len, s_val.len, 0x7f, p, + trec, tot_len) + || tfw_cache_h2_copy_str(&ce->hdr_len, p, trec, &s_val, + tot_len)) + return -ENOMEM; + + CSTR_WRITE_HDR(0, ce->hdr_len - prev_len); } - return copied; + T_DBG3("%s: p=[%p], trec=[%p], ce->hdr_len='%u', tot_len='%zu'\n", + __func__, *p, *trec, ce->hdr_len, *tot_len); + + return ce->hdr_len - init_len; +} + +static long +tfw_cache_h2_add_hdr(TfwCacheEntry *ce, char **p, TdbVRec **trec, + unsigned short st_idx, TfwStr *val, size_t *tot_len) +{ + TfwCStr *cs; + unsigned long len; + unsigned int prev_len = ce->hdr_len; + long n = TFW_CSTR_HDRLEN; + + BUG_ON(!st_idx || TFW_STR_EMPTY(val)); + + len = tfw_hpack_int_size(st_idx, 0xF) + + tfw_hpack_int_size(val->len, 0x7F) + + val->len; + + if (unlikely(len >= TFW_CSTR_MAXLEN)) { + T_WARN("Cache: trying to store too big string %lx\n", len); + return -E2BIG; + } + + /* Don't split short strings. */ + if (TFW_CSTR_HDRLEN + len <= L1_CACHE_BYTES) + n += len; + + *p = tdb_entry_get_room(node_db(), trec, *p, n, *tot_len); + if (unlikely(!*p)) { + T_WARN("jCache: cannot allocate TDB space\n"); + return -ENOMEM; + } + + CSTR_MOVE_HDR(); + + if (tfw_cache_h2_copy_int(&ce->hdr_len, st_idx, 0xF, p, trec, tot_len)) + return -ENOMEM; + + if (tfw_cache_h2_copy_int(&ce->hdr_len, val->len, 0x7f, p, trec, + tot_len)) + return -ENOMEM; + + if (tfw_cache_h2_copy_str(&ce->hdr_len, p, trec, val, tot_len)) + return -ENOMEM; + + CSTR_WRITE_HDR(0, ce->hdr_len - prev_len - TFW_CSTR_HDRLEN); + + return ce->hdr_len - prev_len; } /** @@ -1020,13 +1266,30 @@ __set_etag(TfwCacheEntry *ce, TfwHttpResp *resp, long h_off, TdbVRec *h_trec, char *curr_p, TdbVRec **curr_trec) { char *e_p; - size_t len = 0; - TfwStr etag_val, *c, *end, *h = &resp->h_tbl->tbl[TFW_HTTP_HDR_ETAG]; + size_t c_size; TDB *db = node_db(); + size_t len = 0; + unsigned short flags = 0; + TfwStr h_val, *c, *end, *h = &resp->h_tbl->tbl[TFW_HTTP_HDR_ETAG]; + TfwStr s_dummy = {}, s_val = {}; + +#define CHECK_REC_SPACE() \ + while (c_size) { \ + size_t tail = h_trec->len - (e_p - h_trec->data); \ + if (c_size > tail) { \ + c_size -= tail; \ + h_trec = tdb_next_rec_chunk(db, h_trec); \ + e_p = h_trec->data; \ + } else { \ + e_p += c_size; \ + c_size = 0; \ + } \ + } if (TFW_STR_EMPTY(h)) return 0; - etag_val = tfw_str_next_str_val(h); /* not empty after http parser. */ + + tfw_http_msg_srvhdr_val(h, TFW_HTTP_HDR_ETAG, &h_val); /* Update supposed Etag offset to real value. */ /* FIXME: #803 */ @@ -1035,32 +1298,32 @@ __set_etag(TfwCacheEntry *ce, TfwHttpResp *resp, long h_off, TdbVRec *h_trec, h_trec = tdb_next_rec_chunk(db, h_trec); e_p = h_trec->data; } - /* Skip anything that is not a etag value. */ + /* + * Skip anything that is not etag value. Note, since headers are + * stored in cache in HTTP/2 representation (and the name of 'etag' + * header is always statically indexed), we should also skip index + * and value length fields; the 'etag' header has the 34 index in + * HPACK static table, thus we definitely now here, that it will + * occupy 2 bytes (RFC 7541 section 6.2.2). + */ e_p += TFW_CSTR_HDRLEN; - TFW_STR_FOR_EACH_CHUNK(c, h, end) { - size_t c_size = c->len; - - if (c->flags & TFW_STR_VALUE) + tfw_http_hdr_split(h, &s_dummy, &s_val, true); + c_size = 2 + tfw_hpack_int_size(s_val.len, 0x7F); + CHECK_REC_SPACE(); + TFW_STR_FOR_EACH_CHUNK(c, &h_val, end) { + if (c->flags & TFW_STR_VALUE) { + flags = c->flags; break; - while (c_size) { - size_t tail = h_trec->len - (e_p - h_trec->data); - if (c_size > tail) { - c_size -= tail; - h_trec = tdb_next_rec_chunk(db, h_trec); - e_p = h_trec->data; - } - else { - e_p += c_size; - c_size = 0; - } } + c_size = c->len; + CHECK_REC_SPACE(); } for ( ; (c < end) && (c->flags & TFW_STR_VALUE); ++c) len += c->len; /* Create TfWStr that contains only entity-tag value. */ ce->etag.data = e_p; - ce->etag.flags = TFW_STR_CHUNK(&etag_val, 0)->flags; + ce->etag.flags = flags; ce->etag.len = min(len, (size_t)(h_trec->len - (e_p - h_trec->data))); len -= ce->etag.len; @@ -1090,6 +1353,8 @@ __set_etag(TfwCacheEntry *ce, TfwHttpResp *resp, long h_off, TdbVRec *h_trec, } return 0; + +#undef CHECK_REC_SPACE } /** @@ -1137,26 +1402,36 @@ __save_hdr_304_off(TfwCacheEntry *ce, TfwHttpResp *resp, TfwStr *hdr, long off) /** * Copy response skbs to database mapped area. - * @tot_len - total length of actual data to write w/o TfwCStr's etc. + * @tot_len - total length of actual data to write w/o TfwCStr's etc; + * @rph - response reason-phrase to be saved in the cache. * * It's nasty to copy data on CPU, but we can't use DMA for mmaped file * as well as for unaligned memory areas. - * - * TODO Store the cache entries as a templates with placeholders for - * changeable headers. That we can faster build the final answers instead - * of adjusting skb's. */ static int -tfw_cache_copy_resp(TfwCacheEntry *ce, TfwHttpResp *resp, size_t tot_len) +tfw_cache_copy_resp(TfwCacheEntry *ce, TfwHttpResp *resp, TfwStr *rph, + size_t tot_len) { - TfwHttpReq *req = resp->req; - long n, etag_off = 0; + int r, i; char *p; - TdbVRec *trec = &ce->trec, *etag_trec = NULL; - TfwStr *s_line = &resp->h_tbl->tbl[TFW_HTTP_STATUS_LINE]; + unsigned short status_idx; + TfwStr *field, *h, *end1, *end2; TDB *db = node_db(); - TfwStr *field, *h, *end1, *end2, empty = {}; - int r, i; + TdbVRec *trec = &ce->trec, *etag_trec = NULL; + long n, etag_off = 0; + TfwHttpReq *req = resp->req; + TfwGlobal *g_vhost = tfw_vhost_get_global(); + TfwStr h_val, *host = &req->h_tbl->tbl[TFW_HTTP_HDR_HOST]; + TfwStr val_srv = TFW_STR_STRING(TFW_SERVER); + TfwStr val_via = { + .chunks = (TfwStr []) { + { .data = S_VIA_H2_PROTO, .len = SLEN(S_VIA_H2_PROTO) }, + { .data = *this_cpu_ptr(&g_c_buf), + .len = g_vhost->hdr_via_len }, + }, + .len = SLEN(S_VIA_H2_PROTO) + g_vhost->hdr_via_len, + .nchunks = 2 + }; p = (char *)(ce + 1); tot_len -= CE_BODY_SIZE; @@ -1164,7 +1439,13 @@ tfw_cache_copy_resp(TfwCacheEntry *ce, TfwHttpResp *resp, size_t tot_len) /* Write record key (URI + Host header). */ ce->key = TDB_OFF(db->hdr, p); ce->key_len = 0; - TFW_CACHE_REQ_KEYITER(field, req, end1, h, end2) { + + /* + * Get 'host' header value (from HTTP/2 or HTTP/1.1 request) for + * strict comparison. + */ + tfw_http_msg_clnthdr_val(req, host, TFW_HTTP_HDR_HOST, &h_val); + TFW_CACHE_REQ_KEYITER(field, &req->uri_path, &h_val, end1, h, end2) { if ((n = tfw_cache_strcpy_lc(&p, &trec, field, tot_len)) < 0) { T_ERR("Cache: cannot copy request key\n"); return -ENOMEM; @@ -1173,58 +1454,108 @@ tfw_cache_copy_resp(TfwCacheEntry *ce, TfwHttpResp *resp, size_t tot_len) tot_len -= n; ce->key_len += n; } + /* Request method is a part of the cache record key. */ ce->method = req->method; + /* Write ':status' pseudo-header. */ ce->status = TDB_OFF(db->hdr, p); - if ((n = tfw_cache_strcpy_eol(&p, &trec, s_line, &tot_len, 1)) < 0) { - T_ERR("Cache: cannot copy HTTP status line\n"); - return -ENOMEM; + status_idx = tfw_h2_pseudo_index(resp->status); + if (status_idx) { + TfwHPackInt hp_idx; + TfwStr str = {}; + + write_int(status_idx, 0x7F, 0x80, &hp_idx); + str.data = hp_idx.buf; + str.len = hp_idx.sz; + + if (tfw_cache_h2_copy_str(&ce->status_len, &p, &trec, &str, + &tot_len)) + return -ENOMEM; + } else { + char buf[H2_STAT_VAL_LEN]; + TfwStr str = { + .data = buf, + .len = H2_STAT_VAL_LEN + }; + + /* + * If the ':status' pseudo-header is not fully indexed, set + * the default static index (8) just for the name. + */ + if (tfw_cache_h2_copy_int(&ce->status_len, 8, 0xF, &p, &trec, + &tot_len) + || tfw_cache_h2_copy_int(&ce->status_len, H2_STAT_VAL_LEN, + 0x7f, &p, &trec, &tot_len)) + return -ENOMEM; + + if (!tfw_ultoa(resp->status, str.data, H2_STAT_VAL_LEN)) + return -E2BIG; + + if (tfw_cache_h2_copy_str(&ce->status_len, &p, &trec, &str, + &tot_len)) + return -ENOMEM; } - ce->status_len += n; + + if (tfw_cache_h2_copy_str(&ce->rph_len, &p, &trec, rph, &tot_len)) + return -ENOMEM; ce->hdrs = TDB_OFF(db->hdr, p); ce->hdr_len = 0; ce->hdr_num = resp->h_tbl->off; FOR_EACH_HDR_FIELD_FROM(field, end1, resp, TFW_HTTP_HDR_REGULAR) { - bool hdr_304 = false; - - /* Skip hop-by-hop headers. */ - if (!(field->flags & TFW_STR_HBH_HDR)) { - h = field; - } else if (field - resp->h_tbl->tbl < TFW_HTTP_HDR_RAW) { - h = ∅ - } else { + int hid = field - resp->h_tbl->tbl; + /* + * Skip hop-by-hop headers. Also skip 'Server' header (with + * possible duplicates), since we will substitute it with our + * version of this header. + */ + if ((field->flags & TFW_STR_HBH_HDR) + || hid == TFW_HTTP_HDR_SERVER + || TFW_STR_EMPTY(field)) + { --ce->hdr_num; continue; } - if (field - resp->h_tbl->tbl == TFW_HTTP_HDR_ETAG) { - /* Must be updated after tfw_cache_copy_hdr(). */ + + if (hid == TFW_HTTP_HDR_ETAG) { + /* Must be updated after tfw_cache_h2_copy_hdr(). */ etag_off = TDB_OFF(db->hdr, p); etag_trec = trec; } - hdr_304 = __save_hdr_304_off(ce, resp, field, - TDB_OFF(db->hdr, p)); - n = tfw_cache_copy_hdr(&p, &trec, h, &tot_len); - if (n < 0) { - T_ERR("Cache: cannot copy HTTP header\n"); - return -ENOMEM; - } else if (hdr_304) { - ce->hdr_len_304 += n; - } - ce->hdr_len += n; + __save_hdr_304_off(ce, resp, field, TDB_OFF(db->hdr, p)); + + n = tfw_cache_h2_copy_hdr(ce, &p, &trec, field, &tot_len); + if (unlikely(n < 0)) + return n; } + /* Add 'server' header. */ + n = tfw_cache_h2_add_hdr(ce, &p, &trec, 54, &val_srv, &tot_len); + if (unlikely(n < 0)) + return n; + + /* Add 'via' header. */ + memcpy_fast(__TFW_STR_CH(&val_via, 1)->data, g_vhost->hdr_via, + g_vhost->hdr_via_len); + n = tfw_cache_h2_add_hdr(ce, &p, &trec, 60, &val_via, &tot_len); + if (unlikely(n < 0)) + return n; + + ce->hdr_num += 2; + /* Write HTTP response body. */ ce->body = TDB_OFF(db->hdr, p); - n = tfw_cache_strcpy_eol(&p, &trec, &resp->body, &tot_len, - test_bit(TFW_HTTP_B_CHUNKED, resp->flags)); - if (n < 0) { + r = tfw_cache_h2_copy_str(&ce->body_len, &p, &trec, &resp->body, + &tot_len); + if (unlikely(r)) { T_ERR("Cache: cannot copy HTTP body\n"); return -ENOMEM; } - BUG_ON(tot_len != 0); + + if (WARN_ON_ONCE(tot_len != 0)) + return -EINVAL; ce->version = resp->version; tfw_http_copy_flags(ce->hmflags, resp->flags); @@ -1271,56 +1602,121 @@ tfw_cache_copy_resp(TfwCacheEntry *ce, TfwHttpResp *resp, size_t tot_len) return 0; } -static size_t +static long __cache_entry_size(TfwHttpResp *resp) { + TfwStr host_val, *hdr, *hdr_end; TfwHttpReq *req = resp->req; - size_t size = CE_BODY_SIZE; - TfwStr *h, *hdr, *hdr_end, *dup, *dup_end, empty = {}; + long size, res_size = CE_BODY_SIZE; + TfwStr *host = &req->h_tbl->tbl[TFW_HTTP_HDR_HOST]; + unsigned long via_sz = SLEN(S_VIA_H2_PROTO) + + tfw_vhost_get_global()->hdr_via_len; /* Add compound key size */ - size += req->uri_path.len; - size += req->h_tbl->tbl[TFW_HTTP_HDR_HOST].len; + res_size += req->uri_path.len; + tfw_http_msg_clnthdr_val(req, host, TFW_HTTP_HDR_HOST, &host_val); + res_size += host_val.len; + + /* + * Add the length of ':status' pseudo-header: one byte if fully indexed, + * or five bytes (one byte for name index, one for status code length + * and three for status code itself) if only name is indexed. + */ + ++res_size; + res_size += (1 + H2_STAT_VAL_LEN) * !tfw_h2_pseudo_index(resp->status); /* Add all the headers size */ FOR_EACH_HDR_FIELD_FROM(hdr, hdr_end, resp, TFW_HTTP_HDR_REGULAR) { - /* Skip hop-by-hop headers. */ - if (!(hdr->flags & TFW_STR_HBH_HDR)) - h = hdr; - else if (hdr - resp->h_tbl->tbl < TFW_HTTP_HDR_RAW) - h = ∅ - else + TfwStr *d, *d_end; + int hid = hdr - resp->h_tbl->tbl; + /* + * Skip hop-by-hop headers. Also skip 'Server' header (with + * possible duplicates), since we will substitute it with our + * version of this header. + */ + if ((hdr->flags & TFW_STR_HBH_HDR) + || hid == TFW_HTTP_HDR_SERVER + || TFW_STR_EMPTY(hdr)) continue; - if (!TFW_STR_DUP(h)) { - size += sizeof(TfwCStr); - size += h->len ? (h->len + SLEN(S_CRLF)) : 0; + size = sizeof(TfwCStr); + + if (!TFW_STR_DUP(hdr)) { + TfwStr s_nm = {}, s_val = {}; + + tfw_http_hdr_split(hdr, &s_nm, &s_val, true); + size += tfw_h2_hdr_size(s_nm.len, s_val.len, + hdr->hpack_idx); } else { - size += sizeof(TfwCStr); - TFW_STR_FOR_EACH_DUP(dup, h, dup_end) { + TFW_STR_FOR_EACH_DUP(d, hdr, d_end) { + TfwStr s_nm = {}, s_val = {}; + size += sizeof(TfwCStr); - size += dup->len + SLEN(S_CRLF); + tfw_http_hdr_split(d, &s_nm, &s_val, true); + size += tfw_h2_hdr_size(s_nm.len, s_val.len, + d->hpack_idx); } } + + if (unlikely(size >= TFW_CSTR_MAXLEN)) + goto err; + res_size += size; } - /* Add status line length + CRLF */ - size += resp->h_tbl->tbl[TFW_HTTP_STATUS_LINE].len + SLEN(S_CRLF); + /* + * Add the length of our version of 'Server' header and 'Via' header. + * Note, that we need two bytes for static index, since in the first + * byte we have only four available bits for the index (we do not use + * dynamic indexing during headers storing into the cache, thus `without + * indexing` code must be set in the first index byte, see RFC 7541 + * section 6.2.2 for details), and the 'Server' (as well as 'Via') + * static index doesn't fit to that space. + */ + res_size += sizeof(TfwCStr); + res_size += 2; + res_size += tfw_hpack_int_size(SLEN(TFW_SERVER), 0x7F); + res_size += SLEN(TFW_SERVER); + + res_size += sizeof(TfwCStr); + res_size += 2; + res_size += tfw_hpack_int_size(via_sz, 0x7F); + res_size += via_sz; - /* Add body size accounting CRLF after the last chunk */ - size += resp->body.len; - if (test_bit(TFW_HTTP_B_CHUNKED, resp->flags)) - size += SLEN(S_CRLF); + /* Add body size. */ + res_size += resp->body.len; - return size; + return res_size; +err: + T_WARN("Cache: trying to store too big string %ld\n", size); + return -E2BIG; } static void __cache_add_node(TDB *db, TfwHttpResp *resp, unsigned long key) { + size_t len; TfwCacheEntry *ce; - size_t data_len = __cache_entry_size(resp); - size_t len = data_len; + TfwStr rph, *s_line; + long data_len = __cache_entry_size(resp); + + if (unlikely(data_len < 0)) + return; + + /* + * We need to save the reason-phrase for the case of HTTP/1.1-response + * creation from cache. Note, that reason-phrase is always the last part + * of status-line with the TFW_STR_VALUE flag set. + */ + s_line = &resp->h_tbl->tbl[TFW_HTTP_STATUS_LINE]; + rph = tfw_str_next_str_val(s_line); + if (WARN_ON_ONCE(TFW_STR_EMPTY(&rph))) + return; + + data_len += rph.len; + len = data_len; + + T_DBG3("%s: db=[%p] resp=[%p], req=[%p], key='%lu', data_len='%ld'\n", + __func__, db, resp, resp->req, key, data_len); /* TODO #788: revalidate existing entries before inserting a new one. */ @@ -1334,13 +1730,11 @@ __cache_add_node(TDB *db, TfwHttpResp *resp, unsigned long key) if (!ce) return; - T_DBG3("cache db=%p resp=%p/req=%p/ce=%p: alloc_len=%lu\n", - db, resp, resp->req, ce, len); + T_DBG3("%s: ce=[%p], alloc_len='%lu'\n", __func__, ce, len); - if (tfw_cache_copy_resp(ce, resp, data_len)) { + if (tfw_cache_copy_resp(ce, resp, &rph, data_len)) { /* TODO delete the probably partially built TDB entry. */ } - } static void @@ -1450,38 +1844,30 @@ tfw_cache_purge_method(TfwHttpReq *req) * See do_tcp_sendpages() as reference. */ static int -tfw_cache_build_resp_body(TDB *db, TfwHttpResp *resp, TdbVRec *trec, - TfwMsgIter *it, char *p) +tfw_cache_build_resp_body(TDB *db, TdbVRec *trec, TfwMsgIter *it, char *p) { - int off, f_size, r; + int r; - if (WARN_ON_ONCE(!it->skb)) + if (WARN_ON_ONCE(!it->skb_head)) return -EINVAL; /* - * If headers perfectly fit allocated skbs, then - * it->skb == it->skb_head, see tfw_msg_iter_next_data_frag(). - * Normally all the headers fit single skb, but these two situations - * can't be distinguished. Start after last fragment of last skb in list. + * If all skbs/frags are used up (see @tfw_http_msg_expand_data()), + * create new skb with empty frags to reference the cached body; + * otherwise, use next empty frag in current skb. */ - if ((it->skb == it->skb_head) || (it->frag == -1)) { - it->skb = ss_skb_peek_tail(&it->skb_head); - it->frag = skb_shinfo(it->skb)->nr_frags; - } - else { - skb_frag_t *frag = &skb_shinfo(it->skb)->frags[it->frag]; - if (skb_frag_size(frag)) - ++it->frag; + if (!it->skb) { + if ((r = tfw_msg_iter_append_skb(it))) + return r; + } else { + ++it->frag; + if (it->frag >= MAX_SKB_FRAGS + && (r = tfw_msg_iter_append_skb(it))) + return r; } BUG_ON(it->frag < 0); - if (it->frag >= MAX_SKB_FRAGS - 1 - && (r = tfw_msg_iter_append_skb(it))) - return r; - while (1) { - if (it->frag == MAX_SKB_FRAGS - && (r = tfw_msg_iter_append_skb(it))) - return r; + int off, f_size; /* TDB keeps data by pages and we can reuse the pages. */ off = (unsigned long)p & ~PAGE_MASK; @@ -1491,22 +1877,74 @@ tfw_cache_build_resp_body(TDB *db, TfwHttpResp *resp, TdbVRec *trec, off, f_size); skb_frag_ref(it->skb, it->frag); ss_skb_adjust_data_len(it->skb, f_size); - - if (__tfw_http_msg_add_str_data((TfwHttpMsg *)resp, - &resp->body, p, f_size, - it->skb)) - return - ENOMEM; ++it->frag; } if (!(trec = tdb_next_rec_chunk(db, trec))) break; BUG_ON(trec && !f_size); p = trec->data; + + if (it->frag == MAX_SKB_FRAGS + && (r = tfw_msg_iter_append_skb(it))) + return r; } return 0; } +static int +tfw_cache_set_hdr_age(TfwHttpResp *resp, TfwCacheEntry *ce) +{ + int r; + size_t digs; + bool to_h2 = TFW_MSG_H2(resp->req); + TfwHttpTransIter *mit = &resp->mit; + struct sk_buff **skb_head = &resp->msg.skb_head; + time_t age = tfw_cache_entry_age(ce); + char cstr_age[TFW_ULTOA_BUF_SIZ] = {0}; + char *name = to_h2 ? "age" : "age" S_DLM; + unsigned int nlen = to_h2 ? SLEN("age") : SLEN("age" S_DLM); + TfwStr h_age = { + .chunks = (TfwStr []){ + { .data = name, .len = nlen }, + {} + }, + .len = nlen, + .nchunks = 2 + }; + + if (!(digs = tfw_ultoa(age, cstr_age, TFW_ULTOA_BUF_SIZ))) { + r = -E2BIG; + goto err; + } + + __TFW_STR_CH(&h_age, 1)->data = cstr_age; + __TFW_STR_CH(&h_age, 1)->len = digs; + h_age.len += digs; + + if (to_h2) { + h_age.hpack_idx = 21; + if ((r = tfw_hpack_encode(resp, &h_age, TFW_H2_TRANS_EXPAND, + false))) + goto err; + } else { + if ((r = tfw_http_msg_expand_data(&mit->iter, skb_head, + &h_age, NULL))) + goto err; + + if ((r = tfw_http_msg_expand_data(&mit->iter, skb_head, + &g_crlf, NULL))) + goto err; + } + + return 0; + +err: + T_WARN("Unable to add Age: header, cached response [%p] dropped" + " (err: %d)\n", resp, r); + return r; +} + /** * Build response that can be sent via TCP socket. * @@ -1529,44 +1967,33 @@ tfw_cache_build_resp_body(TDB *db, TfwHttpResp *resp, TdbVRec *trec, * TODO use iterator and passed skbs to be called from net_tx_action. */ static TfwHttpResp * -tfw_cache_build_resp(TfwHttpReq *req, TfwCacheEntry *ce) +tfw_cache_build_resp(TfwHttpReq *req, TfwCacheEntry *ce, time_t lifetime, + unsigned int stream_id) { int h; - char *p; - TfwStr *s_line; + TfwMsgIter *it; TfwHttpResp *resp; - TdbVRec *trec = &ce->trec; + char *p, *head_ptr; + TfwHttpTransIter *mit; + TfwFrameHdr frame_hdr; TDB *db = node_db(); - TfwMsgIter it; - + unsigned long h_len = 0; + struct sk_buff **skb_head; + TdbVRec *trec = &ce->trec; + unsigned char buf[FRAME_HEADER_SIZE]; + TfwHdrMods *h_mods = tfw_vhost_get_hdr_mods(req->location, req->vhost, + TFW_VHOST_HDRMOD_RESP); /* * The allocated response won't be checked by any filters and * is used for sending response data only, so don't initialize * connection and GFSM fields. */ if (!(resp = tfw_http_msg_alloc_resp(req))) - return NULL; - - /* - * Cached responses need SKBTX_SHARED_FRAG flag if they are going to be - * encrypted, since by default encryption happens in-place. As data - * pages are reused, overwriting will damage the date. - */ - if (tfw_http_msg_setup((TfwHttpMsg *)resp, &it, ce->hdr_len + 2, - TFW_CONN_TLS(req->conn) ? SKBTX_SHARED_FRAG : 0)) - { - goto free; - } + goto out; - /* - * Allocate HTTP headers table of proper size. - * There were no other allocations since the table is allocated, - * so realloc() just grows the table and returns the same pointer. - */ - h = (ce->hdr_num + 2 * TFW_HTTP_HDR_NUM - 1) & ~(TFW_HTTP_HDR_NUM - 1); - p = tfw_pool_realloc(resp->pool, resp->h_tbl, TFW_HHTBL_SZ(1), - TFW_HHTBL_EXACTSZ(h)); - BUG_ON(p != (char *)resp->h_tbl); + /* Copy version information and flags */ + resp->version = ce->version; + tfw_http_copy_flags(resp->flags, ce->hmflags); /* Skip record key until status line. */ for (p = TDB_PTR(db->hdr, ce->status); @@ -1576,53 +2003,119 @@ tfw_cache_build_resp(TfwHttpReq *req, TfwCacheEntry *ce) if (unlikely(!trec)) { T_WARN("Huh, partially stored cache entry (key=%lx)?\n", ce->key); - goto err; + goto free; } - s_line = &resp->h_tbl->tbl[TFW_HTTP_STATUS_LINE]; - if (tfw_cache_write_field(db, &trec, resp, &it, &p, - ce->status_len, s_line)) - goto err; + if (tfw_cache_set_status(db, ce, resp, &trec, &p, &h_len)) + goto free; - resp->h_tbl->off = ce->hdr_num; for (h = TFW_HTTP_HDR_REGULAR; h < ce->hdr_num; ++h) { - TFW_STR_INIT(resp->h_tbl->tbl + h); - if (tfw_cache_build_resp_hdr(db, resp, resp->h_tbl->tbl + h, - &trec, &it, &p)) - goto err; + if (tfw_cache_build_resp_hdr(db, resp, h_mods, &trec, &p, + &h_len)) + goto free; } - if (tfw_http_msg_add_data(&it, (TfwHttpMsg *)resp, &resp->crlf, - &g_crlf)) - goto err; + mit = &resp->mit; + skb_head = &resp->msg.skb_head; + WARN_ON_ONCE(mit->acc_len); + it = &mit->iter; - BUG_ON(p != TDB_PTR(db->hdr, ce->body)); - if (tfw_cache_build_resp_body(db, resp, trec, &it, p)) - goto err; + /* + * Set 'set-cookie' header if needed, for HTTP/2 or HTTP/1.1 + * response. + */ + if (tfw_http_sess_resp_process(resp, true)) + goto free; + /* + * RFC 7234 p.4 Constructing Responses from Caches: + * When a stored response is used to satisfy a request without + * validation, a cache MUST generate an Age header field. + */ + if (tfw_cache_set_hdr_age(resp, ce)) + goto free; - resp->version = ce->version; - tfw_http_copy_flags(resp->flags, ce->hmflags); + if (!TFW_MSG_H2(req)) { + /* + * Set additional headers and final CRLF for HTTP/1.1 + * response. + */ + if (tfw_http_expand_hbh(resp, ce->resp_status) + || tfw_http_set_loc_hdrs((TfwHttpMsg *)resp, req, true) + || (lifetime > ce->lifetime + && tfw_http_expand_stale_warn(resp)) + || (!test_bit(TFW_HTTP_B_HDR_DATE, resp->flags) + && tfw_http_expand_hdr_date(resp)) + || tfw_http_msg_expand_data(it, skb_head, &g_crlf, NULL)) + goto free; + + goto write_body; + } + + /* Set additional headers for HTTP/2 response. */ + if (tfw_h2_resp_add_loc_hdrs(resp, h_mods, true) + || (lifetime > ce->lifetime + && tfw_h2_set_stale_warn(resp)) + || (!test_bit(TFW_HTTP_B_HDR_DATE, resp->flags) + && tfw_h2_add_hdr_date(resp, TFW_H2_TRANS_EXPAND, true))) + goto free; + + h_len += mit->acc_len; + + if (h_len > FRAME_MAX_LENGTH || ce->body_len > FRAME_MAX_LENGTH) + goto free; + + frame_hdr.stream_id = stream_id; + /* + * Set header for HTTP/2 DATA frame, if body part of response + * exists. + */ + if (ce->body_len) { + TfwStr h_body = { + .data = buf, + .len = sizeof(buf) + }; + + frame_hdr.length = ce->body_len; + frame_hdr.type = HTTP2_DATA; + frame_hdr.flags = HTTP2_F_END_STREAM; + tfw_h2_pack_frame_header(buf, &frame_hdr); + + if (tfw_http_msg_expand_data(it, skb_head, &h_body, NULL)) + goto free; + } + + /* Set header for HTTP/2 HEADERS frame. */ + frame_hdr.length = h_len; + frame_hdr.type = HTTP2_HEADERS; + frame_hdr.flags = HTTP2_F_END_HEADERS; + + if (!ce->body_len) + frame_hdr.flags |= HTTP2_F_END_STREAM; + + head_ptr = (*skb_head)->data; + tfw_h2_pack_frame_header(buf, &frame_hdr); + memcpy_fast(head_ptr, buf, sizeof(buf)); + +write_body: + /* Fill skb with body from cache for HTTP/2 or HTTP/1.1 response. */ + BUG_ON(p != TDB_PTR(db->hdr, ce->body)); + if (ce->body_len) { + if (tfw_cache_build_resp_body(db, trec, it, p)) + goto free; + if (!TFW_MSG_H2(req) + && test_bit(TFW_HTTP_B_CHUNKED, resp->flags) + && tfw_http_msg_expand_data(it, skb_head, &g_crlf, NULL)) + goto free; + } return resp; -err: - T_WARN("Cannot use cached response, key=%lx\n", ce->key); free: tfw_http_msg_free((TfwHttpMsg *)resp); - return NULL; -} - -static inline int -tfw_cache_set_hdr_age(TfwHttpMsg *hmresp, TfwCacheEntry *ce) -{ - size_t digs; - char cstr_age[TFW_ULTOA_BUF_SIZ] = {0}; - time_t age = tfw_cache_entry_age(ce); - - if (!(digs = tfw_ultoa(age, cstr_age, TFW_ULTOA_BUF_SIZ))) - return -E2BIG; +out: + T_WARN("Cannot use cached response, key=%lx\n", ce->key); + TFW_INC_STAT_BH(clnt.msgs_otherr); - return tfw_http_msg_hdr_xfrm(hmresp, "Age", sizeof("Age") - 1, - cstr_age, digs, TFW_HTTP_HDR_RAW, 0); + return NULL; } static void @@ -1630,6 +2123,7 @@ cache_req_process_node(TfwHttpReq *req, tfw_http_cache_cb_t action) { TfwCacheEntry *ce = NULL; TfwHttpResp *resp = NULL; + unsigned int id = 0; TDB *db = node_db(); TdbIter iter; time_t lifetime; @@ -1651,23 +2145,35 @@ cache_req_process_node(TfwHttpReq *req, tfw_http_cache_cb_t action) if (!tfw_handle_validation_req(req, ce)) goto put; - if (!(resp = tfw_cache_build_resp(req, ce))) - goto out; /* - * RFC 7234 p.4 Constructing Responses from Caches: - * When a stored response is used to satisfy a request without - * validation, a cache MUST generate an Age header field. + * If the stream for HTTP/2-request is already closed (due to some + * error or just reset from the client side), there is no sense to + * forward request to the server. */ - if (tfw_cache_set_hdr_age((TfwHttpMsg *)resp, ce)) { - T_WARN("Unable to add Age: header, cached response [%p] " - "dropped\n", resp); - TFW_INC_STAT_BH(clnt.msgs_otherr); - tfw_http_msg_free((TfwHttpMsg *)resp); - resp = NULL; - goto out; + if (TFW_MSG_H2(req)) { + id = tfw_h2_stream_id(req); + if (unlikely(!id)) { + tfw_http_conn_msg_free((TfwHttpMsg *)req); + goto put; + } + } + + resp = tfw_cache_build_resp(req, ce, lifetime, id); + /* + * The stream of HTTP/2-request should be closed here since we have + * successfully created the resulting response from cache and will + * send this response to the client (without forwarding request to + * the backend), thus the stream will be finished. + */ + if (resp && TFW_MSG_H2(req)) { + id = tfw_h2_stream_id_close(req, HTTP2_HEADERS, + HTTP2_F_END_STREAM); + if (unlikely(!id)) { + tfw_http_msg_free((TfwHttpMsg *)resp); + tfw_http_conn_msg_free((TfwHttpMsg *)req); + goto put; + } } - if (lifetime > ce->lifetime) - __set_bit(TFW_HTTP_B_RESP_STALE, resp->flags); out: if (!resp && (req->cache_ctl.flags & TFW_HTTP_CC_OIFCACHED)) tfw_http_send_resp(req, 504, "resource not cached"); diff --git a/tempesta_fw/hpack.c b/tempesta_fw/hpack.c index fde33a77ec..34d2b20188 100644 --- a/tempesta_fw/hpack.c +++ b/tempesta_fw/hpack.c @@ -186,7 +186,7 @@ do { \ * variable-length integer greater than defined limit, this is the malformed * request and we should drop the parsing process. */ -#define GET_FLEXIBLE(x, new_state) \ +#define GET_FLEXIBLE_lambda(x, new_state, lambda) \ do { \ unsigned int __m = 0; \ unsigned int __c; \ @@ -194,6 +194,7 @@ do { \ if (src >= last) { \ hp->shift = __m; \ NEXT_STATE(new_state); \ + lambda; \ goto out; \ } \ __c = *src++; \ @@ -206,11 +207,14 @@ do { \ } while (__c > 127); \ } while (0) +#define GET_FLEXIBLE(x, new_state) \ + GET_FLEXIBLE_lambda(x, new_state, {}) + /* Continue decoding after interruption due to absence of the next fragment. * If the variable-length integer greater than defined limit, this is the * malformed request and we should drop the parsing process. */ -#define GET_CONTINUE(x) \ +#define GET_CONTINUE_lambda(x, lambda) \ do { \ unsigned int __m = hp->shift; \ unsigned int __c = *src++; \ @@ -224,6 +228,7 @@ do { \ while (__c > 127) { \ if (src >= last) { \ hp->shift = __m; \ + lambda; \ goto out; \ } \ __c = *src++; \ @@ -234,8 +239,12 @@ do { \ goto out; \ } \ } \ + lambda; \ } while (0) +#define GET_CONTINUE(x) \ + GET_CONTINUE_lambda(x, {}) + #define SET_NEXT() \ do { \ hp->curr += 8; \ @@ -379,6 +388,30 @@ do { \ static unsigned long act_hp_str_n; +void +write_int(unsigned long index, unsigned short max, unsigned short mask, + TfwHPackInt *__restrict res_idx) +{ + unsigned int size = 1; + unsigned char *dst = res_idx->buf; + + if (likely(index < max)) { + index |= mask; + } + else { + ++size; + *dst++ = max | mask; + index -= max; + while (index > 0x7F) { + ++size; + *dst++ = (index & 0x7F) | 0x80; + index >>= 7; + } + } + *dst = index; + res_idx->sz = size; +} + static inline TfwStr * tfw_hpack_exp_hdr(TfwPool *__restrict pool, unsigned long len, TfwMsgParseIter *__restrict it) @@ -1158,6 +1191,14 @@ tfw_hpack_hdr_set(TfwHPack *__restrict hp, TfwHttpReq *__restrict req, done: switch (entry->tag) { case TFW_TAG_HDR_H2_METHOD: + if (hp->index == 2) { + req->method = TFW_HTTP_METH_GET; + } else if (hp->index == 3) { + req->method = TFW_HTTP_METH_POST; + } else { + WARN_ON_ONCE(1); + return T_DROP; + } parser->_hdr_tag = TFW_HTTP_HDR_H2_METHOD; break; case TFW_TAG_HDR_H2_SCHEME: @@ -1585,6 +1626,405 @@ tfw_hpack_decode(TfwHPack *__restrict hp, unsigned char *__restrict src, return r; } +/* + * Modified version of HPACK decoder FSM - for cache entries processing, + * HTTP/2-headers decoding (either into HTTP/2 or HTTP/1.1 format) and skb + * expanding at once; only static indexing is allowed, no service HPACK codes, + * no Huffman decoding and no parsing; only a limited subset of HPACK decoder + * FSM states is used. + */ +int +tfw_hpack_cache_decode_expand(TfwHPack *__restrict hp, + TfwHttpResp *__restrict resp, + unsigned char *__restrict src, unsigned long n, + TfwDecodeCacheIter *__restrict dc_iter) +{ + unsigned char c; + unsigned int state; + int r = T_OK; + TfwStr exp_str = {}; + TfwHttpTransIter *mit = &resp->mit; + TfwMsgIter *it = &mit->iter; + bool h2_mode = TFW_MSG_H2(resp->req); + const unsigned char *prev, *last = src + n; + struct sk_buff **skb_head = &resp->msg.skb_head; + +#define GET_NEXT_DATA(cond) \ +do { \ + if (unlikely(cond)) \ + goto out; \ +} while (0) + +#define FIXUP_DATA(str, data, len) \ + if (__tfw_http_msg_add_str_data((TfwHttpMsg *)resp, str, data, \ + len, NULL)) \ + { \ + r = T_DROP; \ + goto out; \ + } + +#define FIXUP_H2_DATA(str, data, len) \ +do { \ + if (h2_mode) \ + FIXUP_DATA(str, data, len); \ +} while (0) + +#define EXPAND_STR_DATA(str) \ +do { \ + if (tfw_http_msg_expand_data(it, skb_head, str, NULL)) { \ + r = T_DROP; \ + goto out; \ + } \ + dc_iter->acc_len += (str)->len; \ +} while (0) + +#define EXPAND_DATA(ptr, length) \ +do { \ + exp_str.data = ptr; \ + exp_str.len = length; \ + EXPAND_STR_DATA(&exp_str); \ +} while (0) + +#define EXPAND_H2_DATA(data, len) \ +do { \ + if (h2_mode) \ + EXPAND_DATA(data, len); \ +} while (0) + + WARN_ON_ONCE(!n); + + state = hp->state; + + T_DBG3("%s: header processing, n=%lu, to_parse=%lu, state=%d\n", + __func__, n, last - src, state); + + switch (state & HPACK_STATE_MASK) { + case HPACK_STATE_READY: + prev = src; + c = *src++; + + /* + * We use only static indexing during headers storing + * into the cache, thus `without indexing` code must be + * always set in the first index byte (RFC 7541 section + * 6.2.2) of cached response; besides, since response + * regular headers have no full indexes in HPACK static + * table, only header's name is allowed to be indexed. + */ + if (WARN_ON_ONCE(c & 0xF0)) { + r = T_DROP; + goto out; + } + + T_DBG3("%s: reference with value...\n", __func__); + + hp->index = c & 0x0F; + if (hp->index == 0x0F) { + GET_FLEXIBLE_lambda(hp->index, + HPACK_STATE_INDEX, { + FIXUP_H2_DATA(&dc_iter->h2_data, src, + src - prev); + }); + } + + T_DBG3("%s: name index: %lu\n", __func__, hp->index); + + FIXUP_H2_DATA(&dc_iter->h2_data, src, src - prev); + + NEXT_STATE(hp->index + ? HPACK_STATE_INDEXED_NAME_TEXT + : HPACK_STATE_NAME); + + GET_NEXT_DATA(src >= last); + + if (hp->index) + goto get_indexed_name; + + /* Fall through. */ + + case HPACK_STATE_NAME: + prev = src; + c = *src++; + + T_DBG3("%s: decode header name length...\n", __func__); + WARN_ON_ONCE(hp->length); + WARN_ON_ONCE(c & 0x80); + + hp->length = c & 0x7F; + if (unlikely(hp->length == 0x7F)) { + GET_FLEXIBLE_lambda(hp->length, + HPACK_STATE_NAME_LENGTH, { + FIXUP_H2_DATA(&dc_iter->h2_data, src, + src - prev); + }); + } + else if (unlikely(hp->length == 0)) { + r = T_DROP; + goto out; + } + + T_DBG3("%s: name length: %lu\n", __func__, hp->length); + + FIXUP_H2_DATA(&dc_iter->h2_data, src, src - prev); + + NEXT_STATE(HPACK_STATE_NAME_TEXT); + + GET_NEXT_DATA(src >= last); + + goto get_name_text; + + case HPACK_STATE_INDEXED_NAME_TEXT: + { + const TfwHPackEntry *entry; +get_indexed_name: + T_DBG3("%s: decode indexed (%lu) header name...\n", + __func__, hp->index); + if (WARN_ON_ONCE(!hp->index + || hp->index > HPACK_STATIC_ENTRIES)) + { + r = T_DROP; + goto out; + } + + entry = static_table + hp->index - 1; + if (WARN_ON_ONCE(entry->name_num != 1)) { + r = T_DROP; + goto out; + } + + dc_iter->hdr_data.len = entry->name_len; + dc_iter->hdr_data.data = __TFW_STR_CH(entry->hdr, 0)->data; + + goto check_name_text; + + } + case HPACK_STATE_NAME_TEXT: + { + int i; + TfwHdrMods *h_mods; + unsigned long m_len; +get_name_text: + m_len = min((unsigned long)(last - src), hp->length); + + T_DBG3("%s: decoding header name, m_len=%lu\n", __func__, m_len); + + FIXUP_DATA(&dc_iter->hdr_data, src, m_len); + + hp->length -= m_len; + src += m_len; + + GET_NEXT_DATA(hp->length); +check_name_text: + i = 0; + h_mods = dc_iter->h_mods; + WARN_ON_ONCE(dc_iter->desc); + if (h_mods) { + for (; i < h_mods->sz; ++i) { + TfwHdrModsDesc *d = &h_mods->hdrs[i]; + + if (!__hdr_name_cmp(&dc_iter->hdr_data, d->hdr)) + { + dc_iter->desc = d; + break; + } + } + } + + if (dc_iter->desc) { + /* All duplicate headers must be skipped by caller. */ + WARN_ON_ONCE(test_bit(i, mit->found)); + __set_bit(i, mit->found); + if (!TFW_STR_CHUNK(dc_iter->desc->hdr, 2)) { + dc_iter->skip = true; + goto out; + } + + } + + if (h2_mode) + EXPAND_STR_DATA(&dc_iter->h2_data); + + EXPAND_STR_DATA(&dc_iter->hdr_data); + TFW_STR_INIT(&dc_iter->hdr_data); + + if (!h2_mode) + EXPAND_DATA(S_DLM, SLEN(S_DLM)); + + T_DBG3("%s: name copied, n=%lu, tail=%lu, hp->length=%lu\n", + __func__, n, last - src, hp->length); + + NEXT_STATE(HPACK_STATE_VALUE); + + GET_NEXT_DATA(src >= last); + + /* Fall through. */ + } + case HPACK_STATE_VALUE: + T_DBG3("%s: decode header value length...\n", __func__); + + prev = src; + c = *src++; + WARN_ON_ONCE(hp->length); + WARN_ON_ONCE(c & 0x80); + + hp->length = c & 0x7F; + if (unlikely(hp->length == 0x7F)) + GET_FLEXIBLE_lambda(hp->length, + HPACK_STATE_VALUE_LENGTH, { + if (!dc_iter->desc) + EXPAND_H2_DATA(src, src - prev); + }); + + T_DBG3("%s: value length: %lu\n", __func__, hp->length); + + if (!dc_iter->desc) + EXPAND_H2_DATA(src, src - prev); + + NEXT_STATE(HPACK_STATE_VALUE_TEXT); + + GET_NEXT_DATA(src >= last); + + /* Fall through. */ + + case HPACK_STATE_VALUE_TEXT: + { + unsigned long m_len; +get_value_text: + T_DBG3("%s: decode header value...\n", __func__); + m_len = min((unsigned long)(last - src), hp->length); + + if (dc_iter->desc && dc_iter->desc->append && h2_mode) { + /* + * If the header value must be appended, we need to + * collect the value for HTTP/2-header, since it should + * be re-encoded in this case. + */ + FIXUP_DATA(&dc_iter->hdr_data, src, m_len); + } + else if (!dc_iter->desc || dc_iter->desc->append) { + EXPAND_DATA(src, m_len); + } + + hp->length -= m_len; + src += m_len; + + GET_NEXT_DATA(hp->length); + + if (dc_iter->desc) { + TfwStr *val, *h = dc_iter->desc->hdr; + TfwStr n_val = { + .chunks = (TfwStr []){ + { .data = ", ", .len = 2 }, + { .data = __TFW_STR_CH(h, 2)->data, + .len = __TFW_STR_CH(h, 2)->len } + }, + .len = __TFW_STR_CH(h, 2)->len + 2, + .nchunks = 2 + }; + + dc_iter->skip = true; + + if (h2_mode) { + TfwHPackInt vlen; + + if (dc_iter->desc->append) { + val = &dc_iter->hdr_data; + if (tfw_strcat(resp->pool, val, &n_val)) + { + r = T_DROP; + goto out; + } + } + else { + val = __TFW_STR_CH(&n_val, 1); + } + + write_int(val->len, 0x7F, 0, &vlen); + + EXPAND_DATA(vlen.buf, vlen.sz); + EXPAND_STR_DATA(val); + + break; + } + + val = dc_iter->desc->append + ? &n_val + : __TFW_STR_CH(&n_val, 1); + + EXPAND_STR_DATA(val); + } + + if (!h2_mode) + EXPAND_DATA(S_CRLF, SLEN(S_CRLF)); + + break; + } + case HPACK_STATE_INDEX: + prev = src; + GET_CONTINUE_lambda(hp->index, { + FIXUP_H2_DATA(&dc_iter->h2_data, src, src - prev); + }); + T_DBG3("%s: index finally decoded: %lu\n", __func__, hp->index); + + NEXT_STATE(HPACK_STATE_INDEXED_NAME_TEXT); + + GET_NEXT_DATA(src >= last); + + goto get_indexed_name; + + case HPACK_STATE_NAME_LENGTH: + prev = src; + GET_CONTINUE_lambda(hp->length, { + FIXUP_H2_DATA(&dc_iter->h2_data, src, src - prev); + }); + T_DBG3("%s: name length finally decoded: %lu\n", __func__, + hp->length); + + NEXT_STATE(HPACK_STATE_NAME_TEXT); + + GET_NEXT_DATA(src >= last); + + goto get_name_text; + + case HPACK_STATE_VALUE_LENGTH: + prev = src; + GET_CONTINUE_lambda(hp->length, { + if (!dc_iter->desc) + EXPAND_H2_DATA(src, src - prev); + }); + T_DBG3("%s: value length finally decoded: %lu\n", __func__, + hp->length); + + NEXT_STATE(HPACK_STATE_VALUE_TEXT); + + GET_NEXT_DATA(src >= last); + + goto get_value_text; + + default: + WARN_ON_ONCE(1); + r = T_DROP; + goto out; + } + + T_DBG3("%s: new header added\n", __func__); + + WARN_ON_ONCE(src != last); + + return T_OK; +out: + WARN_ON_ONCE(src > last); + hp->state = state; + return r; + +#undef GET_NEXT_DATA +#undef FIXUP_DATA +#undef FIXUP_H2_DATA +#undef EXPAND_STR_DATA +#undef EXPAND_DATA +#undef EXPAND_H2_DATA +} + /** * ------------------------------------------------------------------------ * HPACK Encoder functionality @@ -1972,6 +2412,36 @@ do { \ #undef SHIFT } +/* + * Copy the header part (i.e. name/value) into @out_buf from @h_field. + * Return pointer on the next position of @out_buf, after the copied data. + * Note that the size of prepared @out_buf must be not less than the + * length of the @h_field. + */ +static char * +tfw_hpack_write(const TfwStr *h_field, char *out_buf) +{ + const TfwStr *c, *end; + + T_DBG3("%s: enter, h_field->len=%lu,\n", __func__, h_field->len); + + if (WARN_ON_ONCE(TFW_STR_EMPTY(h_field))) + return out_buf; + + TFW_STR_FOR_EACH_CHUNK(c, h_field, end) { + if (!c->len) + continue; + + T_DBG3("%s: c->len=%lu, c->data='%.*s'\n", __func__, c->len, + (int)c->len, c->data); + + memcpy_fast(out_buf, c->data, c->len); + out_buf += c->len; + } + + return out_buf; +} + /* * Left rotation of red-black tree. */ @@ -2504,17 +2974,19 @@ tfw_hpack_rbuf_commit(TfwHPackETbl *__restrict tbl, * headers will be evicted from the index table. */ static int -tfw_hpack_add_node(TfwHPackETbl *__restrict tbl, const TfwStr *__restrict hdr, +tfw_hpack_add_node(TfwHPackETbl *__restrict tbl, TfwStr *__restrict hdr, TfwHPackNodeIter *__restrict place, TfwH2TransOp op) { + char *ptr; unsigned long node_size, hdr_len; unsigned short new_size, node_len; unsigned short cur_size = tbl->size, window = tbl->window; - unsigned long nm_len, val_off, val_len; TfwHPackNode *del_list[HPACK_MAX_ENC_EVICTION] = {}; + TfwStr s_nm = {}, s_val = {}; TfwHPackETblIter it = {}; - hdr_len = tfw_h2_msg_hdr_length(hdr, &nm_len, &val_off, &val_len, op); + hdr_len = tfw_http_hdr_split(hdr, &s_nm, &s_val, + op == TFW_H2_TRANS_INPLACE); WARN_ON_ONCE(cur_size > window || window > HPACK_ENC_TABLE_MAX_SIZE); if ((node_size = hdr_len + HPACK_ENTRY_OVERHEAD) > window) { @@ -2609,7 +3081,9 @@ tfw_hpack_add_node(TfwHPackETbl *__restrict tbl, const TfwStr *__restrict hdr, it.last->hdr_len = hdr_len; it.last->rindex = ++tbl->idx_acc; - tfw_h2_msg_hdr_write(hdr, nm_len, val_off, val_len, it.last->hdr); + ptr = tfw_hpack_write(&s_nm, it.last->hdr); + tfw_hpack_write(&s_val, ptr); + tfw_hpack_rbuf_commit(tbl, del_list, place, &it); WARN_ON_ONCE(tbl->rb_len > tbl->size); @@ -2625,7 +3099,7 @@ tfw_hpack_add_node(TfwHPackETbl *__restrict tbl, const TfwStr *__restrict hdr, */ static TfwHPackETblRes tfw_hpack_encoder_index(TfwHPackETbl *__restrict tbl, - const TfwStr *__restrict hdr, + TfwStr *__restrict hdr, unsigned short *__restrict out_index, unsigned long *__restrict flags, TfwH2TransOp op) @@ -2716,30 +3190,8 @@ tfw_hpack_enc_release(TfwHPack *__restrict hp, unsigned long *flags) WARN_ON_ONCE(!atomic64_read(&tbl->guard)); atomic64_dec(&tbl->guard); } -} - -static void -write_int(unsigned long index, unsigned short max, unsigned short mask, - TfwHPackInt *__restrict res_idx) -{ - unsigned int size = 1; - unsigned char *dst = res_idx->buf; - if (likely(index < max)) { - index |= mask; - } - else { - ++size; - *dst++ = max | mask; - index -= max; - while (index > 0x7F) { - ++size; - *dst++ = (index & 0x7F) | 0x80; - index >>= 7; - } - } - *dst = index; - res_idx->sz = size; + __clear_bit(TFW_HTTP_B_H2_TRANS_ENTERED, flags); } static unsigned long @@ -2920,12 +3372,12 @@ tfw_hpack_str_expand_raw(TfwHttpTransIter *mit, TfwMsgIter *it, len_str.data = len.buf; len_str.len = len.sz; - r = tfw_http_msg_expand_data(it, skb_head, &len_str); + r = tfw_http_msg_expand_data(it, skb_head, &len_str, NULL); if (unlikely(r)) return r; mit->acc_len += len_str.len; - r = tfw_http_msg_expand_data(it, skb_head, str); + r = tfw_http_msg_expand_data(it, skb_head, str, NULL); if (unlikely(r)) return r; mit->acc_len += str->len; @@ -3066,7 +3518,8 @@ tfw_hpack_hdr_expand(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, .len = idx->sz, }; - ret = tfw_http_msg_expand_data(iter, skb_head, &idx_str); + ret = tfw_http_msg_expand_data(iter, skb_head, &idx_str, + &mit->start_off); if (unlikely(ret)) return ret; @@ -3079,9 +3532,6 @@ tfw_hpack_hdr_expand(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, if (!hdr) return 0; - if (WARN_ON_ONCE(TFW_STR_PLAIN(hdr))) - return -EINVAL; - if (unlikely(!name_indexed)) { ret = tfw_hpack_str_expand(mit, iter, skb_head, TFW_STR_CHUNK(hdr, 0), NULL); @@ -3095,10 +3545,16 @@ tfw_hpack_hdr_expand(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, * * { name [S_DLM] value1 [value2 [value3 ...]] }. * + * Besides, we can get here the source header which contains only the + * name (e.g. due to creation of headers separately by parts on the + * upper HTTP level, during internal responses generation) - this is the + * valid case for expanding procedure and we should return control + * upstairs in this case - in order the header creation to be continued. + * */ c = TFW_STR_CHUNK(hdr, 1); - if (WARN_ON_ONCE(!c)) - return -EINVAL; + if (!(c = TFW_STR_CHUNK(hdr, 1))) + return 0; if (c->len == SLEN(S_DLM) && *(short *)c->data == *(short *)S_DLM) { c = TFW_STR_CHUNK(hdr, 2); @@ -3137,10 +3593,7 @@ tfw_hpack_hdr_inplace(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, if (!hdr || WARN_ON_ONCE(TFW_STR_PLAIN(hdr) || TFW_STR_DUP(hdr))) return -EINVAL; - r = tfw_http_hdr_split(hdr, &s_name, &s_val); - - if (unlikely(r)) - return r; + tfw_http_hdr_split(hdr, &s_name, &s_val, true); if (unlikely(!name_indexed)) { TfwHPackInt nlen; @@ -3203,14 +3656,14 @@ tfw_hpack_hdr_inplace(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, */ int tfw_hpack_encode(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, - TfwH2TransOp op) + TfwH2TransOp op, bool dyn_indexing) { TfwHPackInt idx; bool st_full_index; unsigned short st_index, index = 0; TfwH2Ctx *ctx = tfw_h2_context(resp->req->conn); TfwHPackETbl *tbl = &ctx->hpack.enc_tbl; - int r = 0; + int r = HPACK_IDX_ST_NOT_FOUND; if (WARN_ON_ONCE(!hdr || TFW_STR_EMPTY(hdr))) return -EINVAL; @@ -3221,7 +3674,7 @@ tfw_hpack_encode(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, T_DBG3("%s: op=%d, st_index=%hu, st_full_index=%d\n", __func__, op, st_index, st_full_index); - if (!st_full_index) { + if (!st_full_index && dyn_indexing) { r = tfw_hpack_encoder_index(tbl, hdr, &index, resp->flags, op); if (r < 0) return r; diff --git a/tempesta_fw/hpack.h b/tempesta_fw/hpack.h index 01209831e5..7e05394444 100644 --- a/tempesta_fw/hpack.h +++ b/tempesta_fw/hpack.h @@ -20,6 +20,8 @@ #ifndef __TFW_HPACK_H__ #define __TFW_HPACK_H__ +#include "http_types.h" + /** * Default allowed size for dynamic tables (in bytes). Note, that for encoder's * dynamic table - this is also the maximum allowed size and the maximum storage @@ -219,10 +221,35 @@ typedef struct { (DIV_ROUND_UP(sizeof(unsigned long), 7) + 1) typedef struct { - unsigned int sz; - unsigned char buf[HPACK_MAX_INT]; + unsigned int sz; + unsigned char buf[HPACK_MAX_INT]; } TfwHPackInt; + +/** + * Iterator for the message headers decoding from HTTP/2-cache. + * + * @h_mods - pointer to the headers configured to be changed; + * @skip - flag to skip particular cached data in order to switch + * between HTTP/2 and HTTP/1.1 resulting representation during + * decoding from HTTP/2-cache; + * @__off - offset to reinitialize the iterator non-persistent part; + * @desc - pointer to the found header configured to be changed; + * @acc_len - accumulated length of the resulting headers part of the + * response; + * @hdr_data - header's data currently received from cache; + * @h2_data - HTTP/2-specific data currently received from cache. + */ +typedef struct { + TfwHdrMods *h_mods; + bool skip; + char __off[0]; + TfwHdrModsDesc *desc; + unsigned long acc_len; + TfwStr hdr_data; + TfwStr h2_data; +} TfwDecodeCacheIter; + #define BUFFER_GET(len, it) \ do { \ BUG_ON(!(len)); \ @@ -234,15 +261,39 @@ do { \ (it)->pos, (unsigned long)(it)->pos); \ } while (0) +void write_int(unsigned long index, unsigned short max, unsigned short mask, + TfwHPackInt *__restrict res_idx); int tfw_hpack_init(TfwHPack *__restrict hp, unsigned int htbl_sz); void tfw_hpack_clean(TfwHPack *__restrict hp); int tfw_hpack_encode(TfwHttpResp *__restrict resp, TfwStr *__restrict hdr, - TfwH2TransOp op); + TfwH2TransOp op, bool dyn_indexing); void tfw_hpack_set_rbuf_size(TfwHPackETbl *__restrict tbl, unsigned short new_size); int tfw_hpack_decode(TfwHPack *__restrict hp, unsigned char *__restrict src, unsigned long n, TfwHttpReq *__restrict req, unsigned int *__restrict parsed); +int tfw_hpack_cache_decode_expand(TfwHPack *__restrict hp, + TfwHttpResp *__restrict resp, + unsigned char *__restrict src, unsigned long n, + TfwDecodeCacheIter *__restrict cd_iter); void tfw_hpack_enc_release(TfwHPack *__restrict hp, unsigned long *flags); +static inline unsigned int +tfw_hpack_int_size(unsigned long index, unsigned short max) +{ + unsigned int size = 1; + + if (likely(index < max)) + return size; + + ++size; + index -= max; + while (index > 0x7F) { + ++size; + index >>= 7; + } + + return size; +} + #endif /* __TFW_HPACK_H__ */ diff --git a/tempesta_fw/http.c b/tempesta_fw/http.c index 7222054bbe..8c29ea3562 100644 --- a/tempesta_fw/http.c +++ b/tempesta_fw/http.c @@ -112,14 +112,14 @@ #define S_H2_AUTH ":authority" #define S_H2_PATH ":path" #define S_H2_STAT ":status" -#define H2_STAT_VAL_LEN 3 #define T_WARN_ADDR_STATUS(msg, addr_ptr, print_port, status) \ TFW_WITH_ADDR_FMT(addr_ptr, print_port, addr_str, \ T_WARN("%s, status %d: %s\n", \ msg, status, addr_str)) -#define RESP_BUF_LEN 128 +#define RESP_BUF_LEN 128 + static DEFINE_PER_CPU(char[RESP_BUF_LEN], g_buf); int ghprio; /* GFSM hook priority. */ @@ -135,9 +135,7 @@ static struct { #define S_CRLFCRLF "\r\n\r\n" #define S_HTTP "http://" #define S_HTTPS "https://" -#define S_VERSION11 "HTTP/1.1" -#define S_0 S_VERSION11 " " #define S_200 "HTTP/1.1 200 OK" #define S_302 "HTTP/1.1 302 Found" #define S_304 "HTTP/1.1 304 Not Modified" @@ -151,17 +149,17 @@ static struct { #define S_504 "HTTP/1.1 504 Gateway Timeout" #define S_XFF "x-forwarded-for" +#define S_WARN "warning" #define S_F_HOST "host: " #define S_F_DATE "date: " #define S_F_CONTENT_LENGTH "content-length: " #define S_F_CONTENT_TYPE "content-type: " -#define S_F_LOCATION "location: " #define S_F_CONNECTION "connection: " #define S_F_ETAG "etag: " #define S_F_RETRY_AFTER "retry-after: " #define S_F_SERVER "server: " -#define S_F_VIA "via: " + #define S_V_DATE "Sun, 06 Nov 1994 08:49:37 GMT" #define S_V_CONTENT_LENGTH "9999" @@ -169,6 +167,7 @@ static struct { #define S_V_CONN_KA "keep-alive" #define S_V_RETRY_AFTER "10" #define S_V_MULTIPART "multipart/form-data; boundary=" +#define S_V_WARN "110 - Response is stale" #define S_H_CONN_KA S_F_CONNECTION S_V_CONN_KA S_CRLFCRLF #define S_H_CONN_CLOSE S_F_CONNECTION S_V_CONN_CLOSE S_CRLFCRLF @@ -414,6 +413,155 @@ tfw_http_prep_date(char *buf) tfw_http_prep_date_from(buf, tfw_current_timestamp()); } +int +tfw_h2_prep_redirect(TfwHttpResp *resp, unsigned short status, TfwStr *rmark, + TfwStr *cookie, TfwStr *body) +{ + int r; + TfwHPackInt vlen; + TfwFrameHdr frame_hdr; + unsigned int stream_id; + unsigned long hdrs_len, loc_val_len; + unsigned char buf[FRAME_HEADER_SIZE]; + TfwHttpReq *req = resp->req; + TfwHttpTransIter *mit = &resp->mit; + TfwMsgIter *iter = &mit->iter; + struct sk_buff **skb_head = &resp->msg.skb_head; + static TfwStr h_loc = TFW_STR_STRING(S_LOCATION); + static TfwStr proto = TFW_STR_STRING(S_HTTPS); + static TfwStr h_sc = TFW_STR_STRING(S_SET_COOKIE); + TfwStr host, *host_ptr = &host, s_vlen = {}; + + stream_id = tfw_h2_stream_id_close(req, HTTP2_HEADERS, + HTTP2_F_END_STREAM); + if (unlikely(!stream_id)) + return -ENOENT; + + frame_hdr.stream_id = stream_id; + + /* Set HTTP/2 ':status' pseudo-header. */ + mit->start_off = FRAME_HEADER_SIZE; + r = tfw_h2_resp_status_write(resp, status, TFW_H2_TRANS_EXPAND, false); + if (unlikely(r)) + return r; + + /* Add 'date' header. */ + r = tfw_h2_add_hdr_date(resp, TFW_H2_TRANS_EXPAND, false); + if (unlikely(r)) + return r; + + /* Add 'location' header (possibly, with redirection mark). */ + h_loc.hpack_idx = 46; + r = tfw_hpack_encode(resp, &h_loc, TFW_H2_TRANS_EXPAND, false); + if (unlikely(r)) + return r; + + if (req->host.len) + host_ptr = &req->host; + else + __h2_msg_hdr_val(&req->h_tbl->tbl[TFW_HTTP_HDR_HOST], &host); + + loc_val_len = req->uri_path.len; + loc_val_len += host_ptr->len ? host_ptr->len + proto.len : 0; + loc_val_len += rmark->len; + + write_int(loc_val_len, 0x7F, 0, &vlen); + s_vlen.data = vlen.buf; + s_vlen.len = vlen.sz; + + r = tfw_http_msg_expand_data(iter, skb_head, &s_vlen, NULL); + if (unlikely(r)) + return r; + + if (host_ptr->len) { + r = tfw_http_msg_expand_data(iter, skb_head, &proto, NULL); + if (unlikely(r)) + return r; + + r = tfw_http_msg_expand_data(iter, skb_head, host_ptr, NULL); + if (unlikely(r)) + return r; + } + + if (rmark->len) { + r = tfw_http_msg_expand_data(iter, skb_head, rmark, NULL); + if (unlikely(r)) + return r; + } + + r = tfw_http_msg_expand_data(iter, skb_head, &req->uri_path, NULL); + if (unlikely(r)) + return r; + + hdrs_len = s_vlen.len + loc_val_len; + + /* Add 'set-cookie' header. */ + h_sc.hpack_idx = 55; + r = tfw_hpack_encode(resp, &h_sc, TFW_H2_TRANS_EXPAND, false); + if (unlikely(r)) + return r; + + write_int(cookie->len, 0x7F, 0, &vlen); + s_vlen.data = vlen.buf; + s_vlen.len = vlen.sz; + + r = tfw_http_msg_expand_data(iter, skb_head, &s_vlen, NULL); + if (unlikely(r)) + return r; + + r = tfw_http_msg_expand_data(iter, skb_head, cookie, NULL); + if (unlikely(r)) + return r; + + hdrs_len += s_vlen.len + cookie->len; + + /* + * Set message body (if exists) into DATA frame header. In this case + * 'content-length' header also must be added into HEADERS frame. + */ + if (body) { + TfwStr f_hdr = { + .data = buf, + .len = sizeof(buf) + }; + + if (WARN_ON_ONCE(!body->len)) + return -EINVAL; + + if (body->len > FRAME_MAX_LENGTH) + return -E2BIG; + + frame_hdr.length = body->len; + frame_hdr.type = HTTP2_DATA; + frame_hdr.flags = HTTP2_F_END_STREAM; + tfw_h2_pack_frame_header(buf, &frame_hdr); + + r = tfw_http_msg_expand_data(iter, skb_head, &f_hdr, NULL); + if (unlikely(r)) + return r; + + r = tfw_http_msg_expand_data(iter, skb_head, body, NULL); + if (unlikely(r)) + return r; + } + + hdrs_len += mit->acc_len; + if (hdrs_len > FRAME_MAX_LENGTH) + return -E2BIG; + + /* Set frame header for HEADERS. */ + frame_hdr.length = hdrs_len; + frame_hdr.type = HTTP2_HEADERS; + frame_hdr.flags = HTTP2_F_END_HEADERS; + if (!body) + frame_hdr.flags |= HTTP2_F_END_STREAM; + + tfw_h2_pack_frame_header(buf, &frame_hdr); + memcpy_fast((*skb_head)->data, buf, sizeof(buf)); + + return 0; +} + #define S_REDIR_302 S_302 S_CRLF #define S_REDIR_503 S_503 S_CRLF #define S_REDIR_GEN " Redirection" S_CRLF @@ -428,11 +576,11 @@ tfw_http_prep_date(char *buf) * The response redirects the client to the same URI as the original request, * but it includes 'Set-Cookie:' header field that sets Tempesta sticky cookie. * If JS challenge is enabled, then body contained JS challenge is provided. - * Body string contains the 'Content-Legth' header, CRLF and body itself. + * Body string contains the 'Content-Length' header, CRLF and body itself. */ int -tfw_http_prep_redirect(TfwHttpMsg *resp, unsigned short status, TfwStr *rmark, - TfwStr *cookie, TfwStr *body) +tfw_h1_prep_redirect(TfwHttpResp *resp, unsigned short status, TfwStr *rmark, + TfwStr *cookie, TfwStr *body) { TfwHttpReq *req = resp->req; size_t data_len; @@ -511,7 +659,7 @@ tfw_http_prep_redirect(TfwHttpMsg *resp, unsigned short status, TfwStr *rmark, data_len += req->uri_path.len + h_common_2.len + cookie->len; data_len += cookie_crlf->len + r_end->len; - if (tfw_http_msg_setup(resp, &it, data_len, 0)) + if (tfw_http_msg_setup((TfwHttpMsg *)resp, &it, data_len, 0)) return TFW_BLOCK; tfw_http_prep_date(__TFW_STR_CH(&h_common_1, 1)->data); @@ -542,14 +690,13 @@ tfw_http_prep_redirect(TfwHttpMsg *resp, unsigned short status, TfwStr *rmark, #define S_304_PART_01 S_304 S_CRLF #define S_304_KEEP S_F_CONNECTION S_V_CONN_KA S_CRLF #define S_304_CLOSE S_F_CONNECTION S_V_CONN_CLOSE S_CRLF + /* - * HTTP 304 response: Not Modified. + * Preparing 304 response (Not Modified) for HTTP/1.1-client. */ int -tfw_http_prep_304(TfwHttpMsg *resp, TfwHttpReq *req, TfwMsgIter *it, - size_t hdrs_size) +tfw_http_prep_304(TfwHttpReq *req, struct sk_buff **skb_head, TfwMsgIter *it) { - size_t data_len = SLEN(S_304_PART_01); int ret = 0; static TfwStr rh = { .data = S_304_PART_01, .len = SLEN(S_304_PART_01) }; @@ -565,21 +712,19 @@ tfw_http_prep_304(TfwHttpMsg *resp, TfwHttpReq *req, TfwMsgIter *it, else if (test_bit(TFW_HTTP_B_CONN_KA, req->flags)) end = &crlf_keep; - /* Add variable part of data length to get the total */ - data_len += hdrs_size; - if (end) - data_len += end->len; - - if (tfw_http_msg_setup(resp, it, data_len, 0)) - return TFW_BLOCK; + ret = tfw_http_msg_expand_data(it, skb_head, &rh, NULL); + if (unlikely(ret)) + return ret; - ret = tfw_msg_write(it, &rh); - if (end) - ret |= tfw_msg_write(it, end); + if (end) { + ret = tfw_http_msg_expand_data(it, skb_head, end, NULL); + if (unlikely(ret)) + return ret; + } T_DBG("Send HTTP 304 response\n"); - return ret ? TFW_BLOCK : TFW_PASS; + return 0; } /* @@ -713,43 +858,17 @@ tfw_http_enum_resp_code(int status) } } -/* - * Static index determination for response ':status' pseudo-header (see RFC - * 7541 Appendix A for details). - */ -static inline unsigned short -tfw_h2_pseudo_index(unsigned short status) -{ - switch (status) { - case 200: - return 8; - case 204: - return 9; - case 206: - return 10; - case 304: - return 11; - case 400: - return 12; - case 404: - return 13; - case 500: - return 14; - default: - return 0; - } -} - /** - * Write HTTP/2 pseudo-header fields. Only ':status' is defined as response - * pseudo-header and all HTTP/2 responses must contain that. - * https://httpwg.org/specs/rfc7540.html#rfc.section.8.1.2.4 + * Write HTTP/2 ':status' pseudo-header. The ':status' is only defined + * pseudo-header for the response and all HTTP/2 responses must contain it. + * https://httpwg.org/specs/rfc7540.html#rfc.section.8.1.2.4. */ -static int -tfw_h2_pseudo_write(TfwHttpResp *resp, TfwH2TransOp op) +int +tfw_h2_resp_status_write(TfwHttpResp *resp, unsigned short status, + TfwH2TransOp op, bool cache) { int ret; - unsigned short index = tfw_h2_pseudo_index(resp->status); + unsigned short index = tfw_h2_pseudo_index(status); char buf[H2_STAT_VAL_LEN]; TfwStr s_hdr = { .chunks = (TfwStr []){ @@ -771,17 +890,16 @@ tfw_h2_pseudo_write(TfwHttpResp *resp, TfwH2TransOp op) s_hdr.flags |= TFW_STR_FULL_INDEX; } - if (!tfw_ultoa(resp->status, __TFW_STR_CH(&s_hdr, 1)->data, - H2_STAT_VAL_LEN)) + if (!tfw_ultoa(status, __TFW_STR_CH(&s_hdr, 1)->data, H2_STAT_VAL_LEN)) return -E2BIG; - if ((ret = tfw_hpack_encode(resp, &s_hdr, op))) + if ((ret = tfw_hpack_encode(resp, &s_hdr, op, !cache))) return ret; return 0; } -static inline void +void tfw_h2_resp_fwd(TfwHttpResp *resp) { TfwHttpReq *req = resp->req; @@ -811,9 +929,7 @@ tfw_h2_send_resp(TfwHttpReq *req, int status, unsigned int stream_id) TfwFrameHdr frame_hdr; TfwHttpTransIter *mit; char *date_val, *data_ptr; - unsigned int len_be; unsigned long nlen, vlen; - unsigned char *dst_len_p, *src_len_p; unsigned char buf[FRAME_HEADER_SIZE]; TfwStr *start, *date, *clen, *srv, *body; TfwH2Ctx *ctx = tfw_h2_context(req->conn); @@ -835,6 +951,8 @@ tfw_h2_send_resp(TfwHttpReq *req, int status, unsigned int stream_id) } } + frame_hdr.stream_id = stream_id; + code = tfw_http_enum_resp_code(status); if (code == RESP_NUM) { T_WARN("Unexpected response error code: [%d]\n", status); @@ -859,24 +977,9 @@ tfw_h2_send_resp(TfwHttpReq *req, int status, unsigned int stream_id) goto err_setup; } - /* Create frame header for HEADERS. Note, that we leave the length of - * HEADERS frame unset, to be filled later (below) after all headers - * will be processed, indexed (if needed) and written into the target - * skbs. */ - frame_hdr.stream_id = stream_id; - frame_hdr.length = 0; - frame_hdr.type = HTTP2_HEADERS; - frame_hdr.flags = HTTP2_F_END_HEADERS; - if (!body->data) - frame_hdr.flags |= HTTP2_F_END_STREAM; - tfw_h2_pack_frame_header(buf, &frame_hdr); - - /* Set frame header and HTTP/2 ':status' pseudo-header. */ - if (tfw_http_msg_expand_data(&resp->mit.iter, skb_head, &f_hdr)) - goto err_setup; - - resp->status = (unsigned short)status; - if (tfw_h2_pseudo_write(resp, TFW_H2_TRANS_EXPAND)) + /* Set HTTP/2 ':status' pseudo-header. */ + mit->start_off = FRAME_HEADER_SIZE; + if (tfw_h2_resp_status_write(resp, status, TFW_H2_TRANS_EXPAND, false)) goto err_setup; /* @@ -893,7 +996,7 @@ tfw_h2_send_resp(TfwHttpReq *req, int status, unsigned int stream_id) __TFW_STR_CH(&hdr, 1)->len = vlen = date->len; hdr.len = nlen + vlen; hdr.hpack_idx = 33; - if (tfw_hpack_encode(resp, &hdr, TFW_H2_TRANS_EXPAND)) + if (tfw_hpack_encode(resp, &hdr, TFW_H2_TRANS_EXPAND, true)) goto err_setup; clen = TFW_STR_CLEN_CH(msg); @@ -904,7 +1007,7 @@ tfw_h2_send_resp(TfwHttpReq *req, int status, unsigned int stream_id) - data_ptr - SLEN(S_CRLF); hdr.len = nlen + vlen; hdr.hpack_idx = 28; - if (tfw_hpack_encode(resp, &hdr, TFW_H2_TRANS_EXPAND)) + if (tfw_hpack_encode(resp, &hdr, TFW_H2_TRANS_EXPAND, true)) goto err_setup; srv = TFW_STR_SRV_CH(msg); @@ -915,19 +1018,12 @@ tfw_h2_send_resp(TfwHttpReq *req, int status, unsigned int stream_id) - data_ptr - SLEN(S_CRLF); hdr.len = nlen + vlen; hdr.hpack_idx = 54; - if (tfw_hpack_encode(resp, &hdr, TFW_H2_TRANS_EXPAND)) + if (tfw_hpack_encode(resp, &hdr, TFW_H2_TRANS_EXPAND, true)) goto err_setup; if (WARN_ON_ONCE(!mit->acc_len)) goto err_setup; - /* - * Get the pointer to head skb data to write the frame's accumulated - * length. We can do this here, since we always allocate new skb in - * @tfw_http_msg_expand_data() with SKB_MAX_HEADER length for the - * head part of skb (also, see description of frame header format in - * RFC 7540 section 4.1). - */ if (WARN_ON_ONCE(mit->acc_len > ctx->rsettings.max_frame_sz)) { /* * TODO #1378: multiple frames might be required here. @@ -939,14 +1035,6 @@ tfw_h2_send_resp(TfwHttpReq *req, int status, unsigned int stream_id) */ goto err_setup; } - len_be = htonl((unsigned int)mit->acc_len); - src_len_p = (unsigned char *)&len_be; - dst_len_p = (*skb_head)->data; - - *(unsigned short *)dst_len_p = *(unsigned short *)++src_len_p; - src_len_p += 2; - dst_len_p += 2; - *dst_len_p = *src_len_p; /* Create and set frame header and set payload for DATA. */ if (body->data) { @@ -955,11 +1043,26 @@ tfw_h2_send_resp(TfwHttpReq *req, int status, unsigned int stream_id) frame_hdr.flags = HTTP2_F_END_STREAM; tfw_h2_pack_frame_header(buf, &frame_hdr); - if (tfw_http_msg_expand_data(&resp->mit.iter, skb_head, &f_hdr) - || tfw_http_msg_expand_data(&resp->mit.iter, skb_head, body)) + if (tfw_http_msg_expand_data(&resp->mit.iter, skb_head, &f_hdr, + NULL) + || tfw_http_msg_expand_data(&resp->mit.iter, skb_head, body, + NULL)) goto err_setup; } + /* + * Set the frame header for HEADERS. Note, that we leave the place for + * it at the beginning of the response - due to @mit->start_off setting + * above (before the first data is written into the target skb). + */ + frame_hdr.length = mit->acc_len; + frame_hdr.type = HTTP2_HEADERS; + frame_hdr.flags = HTTP2_F_END_HEADERS; + if (!body->data) + frame_hdr.flags |= HTTP2_F_END_STREAM; + tfw_h2_pack_frame_header(buf, &frame_hdr); + memcpy_fast((*skb_head)->data, buf, sizeof(buf)); + /* Send resulting HTTP/2 response and release HPACK encoder index. */ tfw_h2_resp_fwd(resp); @@ -2540,6 +2643,36 @@ tfw_http_set_hdr_date(TfwHttpMsg *hm) return r; } +/* + * Expand HTTP response with 'Date:' header field. + */ +int +tfw_http_expand_hdr_date(TfwHttpResp *resp) +{ + int r; + struct sk_buff **skb_head = &resp->msg.skb_head; + TfwHttpTransIter *mit = &resp->mit; + char *date = *this_cpu_ptr(&g_buf); + TfwStr h_date = { + .chunks = (TfwStr []){ + { .data = S_F_DATE, .len = SLEN(S_F_DATE) }, + { .data = date, .len = SLEN(S_V_DATE) }, + { .data = S_CRLF, .len = SLEN(S_CRLF) } + }, + .len = SLEN(S_F_DATE) + SLEN(S_V_DATE) + SLEN(S_CRLF), + .nchunks = 3 + }; + + tfw_http_prep_date_from(date, resp->date); + r = tfw_http_msg_expand_data(&mit->iter, skb_head, &h_date, NULL); + if (r) + T_ERR("Unable to expand resp [%p] with 'Date:' header\n", resp); + else + T_DBG2("Epanded resp [%p] with 'Date:' header\n", resp); + + return r; +} + /** * Connection is to be closed after response for the request @req is forwarded * to the client. Don't process new requests from the client and update @@ -2552,6 +2685,51 @@ tfw_http_req_set_conn_close(TfwHttpReq *req) set_bit(TFW_HTTP_B_CONN_CLOSE, req->flags); } +/** + * Expand HTTP/1.1 response with hop-by-hop headers. It is implied that this + * procedure should be used only for cases when original hop-by-hop headers + * is already removed from the response: e.g. creation HTTP/1.1-response from + * the cache (see also comments for tfw_http_set_hdr_connection(), + * tfw_http_set_hdr_keep_alive() and tfw_http_adjust_resp()). + */ +int +tfw_http_expand_hbh(TfwHttpResp *resp, unsigned short status) +{ + TfwHttpReq *req = resp->req; + TfwHttpTransIter *mit = &resp->mit; + struct sk_buff **skb_head = &resp->msg.skb_head; + bool proxy_close = test_bit(TFW_HTTP_B_CONN_CLOSE, resp->flags) + && (status / 100 == 4); + TfwStr h_conn = { + .chunks = (TfwStr []){ + { .data = S_F_CONNECTION, .len = SLEN(S_F_CONNECTION) }, + {}, + { .data = S_CRLF, .len = SLEN(S_CRLF) } + }, + .len = SLEN(S_F_CONNECTION) + SLEN(S_CRLF), + .nchunks = 3 + }; + + if (unlikely(test_bit(TFW_HTTP_B_CONN_CLOSE, req->flags) + || proxy_close)) + { + __TFW_STR_CH(&h_conn, 1)->data = S_V_CONN_CLOSE; + __TFW_STR_CH(&h_conn, 1)->len = SLEN(S_V_CONN_CLOSE); + h_conn.len += SLEN(S_V_CONN_CLOSE); + } + else if (test_bit(TFW_HTTP_B_CONN_KA, req->flags)) + { + __TFW_STR_CH(&h_conn, 1)->data = S_V_CONN_KA; + __TFW_STR_CH(&h_conn, 1)->len = SLEN(S_V_CONN_KA); + h_conn.len += SLEN(S_V_CONN_KA); + } + + if (unlikely(proxy_close)) + tfw_http_req_set_conn_close(req); + + return tfw_http_msg_expand_data(&mit->iter, skb_head, &h_conn, NULL); +} + /** * Remove Connection header from HTTP message @msg if @conn_flg is zero, * and replace or set a new header value otherwise. @@ -2626,6 +2804,29 @@ tfw_http_set_hdr_keep_alive(TfwHttpMsg *hm, unsigned long conn_flg) } } +/* + * In case if response is stale, we should pass it with a warning. + */ +int +tfw_http_expand_stale_warn(TfwHttpResp *resp) +{ + /* TODO: adjust for #865 */ + struct sk_buff **skb_head = &resp->msg.skb_head; + TfwHttpTransIter *mit = &resp->mit; + TfwStr wh = { + .chunks = (TfwStr []){ + { .data = S_WARN, .len = SLEN(S_WARN) }, + { .data = S_DLM, .len = SLEN(S_DLM) }, + { .data = S_V_WARN, .len = SLEN(S_V_WARN) }, + { .data = S_CRLF, .len = SLEN(S_CRLF) } + }, + .len = SLEN(S_WARN) + SLEN(S_DLM) + SLEN(S_V_WARN) + SLEN(S_CRLF), + .nchunks = 4, + }; + + return tfw_http_msg_expand_data(&mit->iter, skb_head, &wh, NULL); +} + static int tfw_http_add_hdr_via(TfwHttpMsg *hm) { @@ -2682,55 +2883,6 @@ tfw_http_add_x_forwarded_for(TfwHttpMsg *hm) return r; } -static int -tfw_http_set_loc_hdrs(TfwHttpMsg *hm, TfwHttpReq *req) -{ - size_t i; - int mod_type = (hm == (TfwHttpMsg *)req) ? TFW_VHOST_HDRMOD_REQ - : TFW_VHOST_HDRMOD_RESP; - TfwHdrMods *h_mods = tfw_vhost_get_hdr_mods(req->location, req->vhost, - mod_type); - if (!h_mods) - return 0; - - for (i = 0; i < h_mods->sz; ++i) { - TfwHdrModsDesc *d = &h_mods->hdrs[i]; - /* - * Header is stored optimized for HTTP2: without delimiter - * between header and value. Add it as separate chunk as - * required for tfw_http_msg_hdr_xfrm_str. - */ - TfwStr h_mdf = { - .chunks = (TfwStr []){ - {}, - { .data = S_DLM, .len = SLEN(S_DLM) }, - {} - }, - .len = SLEN(S_DLM), - .nchunks = 2 /* header name + delimeter. */ - }; - int r; - - h_mdf.chunks[0] = d->hdr->chunks[0]; - if (d->hdr->nchunks == 2) { - h_mdf.chunks[2] = d->hdr->chunks[1]; - h_mdf.nchunks += 1; - } - h_mdf.len += d->hdr->len; - h_mdf.flags = d->hdr->flags; - h_mdf.eolen += d->hdr->eolen; - r = tfw_http_msg_hdr_xfrm_str(hm, &h_mdf, d->hid, d->append); - if (r) { - T_ERR("can't update location-specific header in msg %p\n", - hm); - return r; - } - T_DBG2("updated location-specific header in msg %p\n", hm); - } - - return 0; -} - /** * Compose Content-Type header field from scratch. * @@ -2783,6 +2935,76 @@ tfw_http_should_validate_post_req(TfwHttpReq *req) return false; } +int +tfw_http_set_loc_hdrs(TfwHttpMsg *hm, TfwHttpReq *req, bool cache) +{ + size_t i; + bool hm_req = (hm == (TfwHttpMsg *)req); + int mod_type = hm_req ? TFW_VHOST_HDRMOD_REQ : TFW_VHOST_HDRMOD_RESP; + TfwHdrMods *h_mods = tfw_vhost_get_hdr_mods(req->location, req->vhost, + mod_type); + BUG_ON(hm_req && cache); + if (!h_mods) + return 0; + + for (i = 0; i < h_mods->sz; ++i) { + int r; + TfwHdrModsDesc *d = &h_mods->hdrs[i]; + /* + * Header is stored optimized for HTTP2: without delimiter + * between header and value. Add it as separate chunk as + * required for tfw_http_msg_hdr_xfrm_str. + */ + TfwStr h_mdf = { + .chunks = (TfwStr []){ + {}, + { .data = S_DLM, .len = SLEN(S_DLM) }, + {} + }, + .len = SLEN(S_DLM), + .nchunks = 2 /* header name + delimeter. */ + }; + + h_mdf.chunks[0] = d->hdr->chunks[0]; + if (d->hdr->nchunks == 2) { + h_mdf.chunks[2] = d->hdr->chunks[1]; + h_mdf.nchunks += 1; + } + h_mdf.len += d->hdr->len; + h_mdf.flags = d->hdr->flags; + h_mdf.eolen += d->hdr->eolen; + + if (!hm_req && cache) { + TfwHttpResp *resp = (TfwHttpResp *)hm; + struct sk_buff **skb_head = &resp->msg.skb_head; + TfwHttpTransIter *mit = &resp->mit; + /* + * Skip the configured header if we have already + * processed it during cache reading, or if the header + * is configured for deletion (without value chunk). + */ + if (test_bit(i, mit->found) || h_mdf.nchunks < 3) + continue; + + r = tfw_http_msg_expand_data(&mit->iter, skb_head, + &h_mdf, NULL); + } else { + r = tfw_http_msg_hdr_xfrm_str(hm, &h_mdf, d->hid, + d->append); + } + + if (r) { + T_ERR("can't update location-specific header in msg %p\n", + hm); + return r; + } + + T_DBG2("updated location-specific header in msg %p\n", hm); + } + + return 0; +} + /** * Adjust the request before proxying it to real server. */ @@ -2808,7 +3030,7 @@ tfw_h1_adjust_req(TfwHttpReq *req) if (r < 0) return r; - r = tfw_http_set_loc_hdrs(hm, req); + r = tfw_http_set_loc_hdrs(hm, req, false); if (r < 0) return r; @@ -2884,7 +3106,7 @@ __h2_req_hdrs(TfwHttpReq *req, const TfwStr *hdr, unsigned int hid, bool append) } } else { - hid = __h2_hdr_lookup(hm, TFW_STR_CHUNK(hdr, 0)); + hid = __http_hdr_lookup(hm, hdr); if (hid == ht->off && !s_val) /* * The raw header not found, and there is nothing @@ -3305,7 +3527,7 @@ tfw_http_adjust_resp(TfwHttpResp *resp) conn_flg = BIT(TFW_HTTP_B_CONN_KA); } - r = tfw_http_sess_resp_process(resp); + r = tfw_http_sess_resp_process(resp, false); if (r < 0) return r; @@ -3325,24 +3547,10 @@ tfw_http_adjust_resp(TfwHttpResp *resp) if (r < 0) return r; - r = tfw_http_set_loc_hdrs(hm, req); + r = tfw_http_set_loc_hdrs(hm, req, false); if (r < 0) return r; - if (test_bit(TFW_HTTP_B_RESP_STALE, resp->flags)) { -#define S_WARN_110 "Warning: 110 - Response is stale" - /* TODO: adjust for #865 */ - TfwStr wh = { - .data = S_WARN_110, - .len = SLEN(S_WARN_110), - .eolen = 2 - }; - r = tfw_http_msg_hdr_add(hm, &wh); - if (r) - return r; -#undef S_WARN_110 - } - if (!test_bit(TFW_HTTP_B_HDR_DATE, resp->flags)) { r = tfw_http_set_hdr_date(hm); if (r < 0) @@ -3536,18 +3744,16 @@ tfw_h2_hdr_map(TfwHttpResp *resp, const TfwStr *hdr, unsigned int id) static int tfw_h2_add_hdr_via(TfwHttpResp *resp) { -#define NM_VIA "via" -#define V_PROTO "2.0 " int r; TfwGlobal *g_vhost = tfw_vhost_get_global(); TfwStr via = { .chunks = (TfwStr []) { - { .data = NM_VIA, .len = SLEN(NM_VIA) }, - { .data = V_PROTO, .len = SLEN(V_PROTO) }, + { .data = S_VIA, .len = SLEN(S_VIA) }, + { .data = S_VIA_H2_PROTO, .len = SLEN(S_VIA_H2_PROTO) }, { .data = *this_cpu_ptr(&g_buf), .len = g_vhost->hdr_via_len }, }, - .len = SLEN(NM_VIA) + SLEN(V_PROTO) + g_vhost->hdr_via_len, + .len = SLEN(S_VIA) + SLEN(S_VIA_H2_PROTO) + g_vhost->hdr_via_len, .nchunks = 3 }; @@ -3556,29 +3762,37 @@ tfw_h2_add_hdr_via(TfwHttpResp *resp) via.hpack_idx = 60; - r = __hdr_h2_add(resp, &via); + r = tfw_hpack_encode(resp, &via, TFW_H2_TRANS_ADD, true); if (unlikely(r)) T_ERR("HTTP/2: unable to add 'via' header (resp=[%p])\n", resp); else T_DBG3("%s: added 'via' header, resp=[%p]\n", __func__, resp); return r; -#undef NM_VIA -#undef V_PROTO } /* * Same as @tfw_http_set_hdr_date(), but intended for usage in HTTP/1.1=>HTTP/2 * transformation. */ -static int -tfw_h2_add_hdr_date(TfwHttpResp *resp) +int +tfw_h2_add_hdr_date(TfwHttpResp *resp, TfwH2TransOp op, bool cache) { int r; char *s_date = *this_cpu_ptr(&g_buf); + TfwStr hdr = { + .chunks = (TfwStr []){ + { .data = "date", .len = SLEN("date") }, + { .data = s_date, .len = SLEN(S_V_DATE) }, + }, + .len = SLEN("date") + SLEN(S_V_DATE), + .nchunks = 2 + }; tfw_http_prep_date_from(s_date, resp->date); - r = tfw_h2_msg_hdr_add(resp, "date", SLEN("date"), s_date, - SLEN(S_V_DATE), TFW_HTTP_HDR_RAW, 33); + + hdr.hpack_idx = 33; + + r = tfw_hpack_encode(resp, &hdr, op, !cache); if (unlikely(r)) T_ERR("HTTP/2: unable to add 'date' header to response" " [%p]\n", resp); @@ -3588,63 +3802,93 @@ tfw_h2_add_hdr_date(TfwHttpResp *resp) return r; } -static inline int +/* + * In case if response is stale, we should pass it with a warning. + */ +int tfw_h2_set_stale_warn(TfwHttpResp *resp) { -#define WARN_NM "warning" -#define WARN_VAL "110 - Response is stale" - if (test_bit(TFW_HTTP_B_RESP_STALE, resp->flags)) { - TfwStr wh = { - .chunks = (TfwStr []){ - { .data = WARN_NM, .len = SLEN(WARN_NM) }, - { .data = WARN_VAL, .len = SLEN(WARN_VAL) } - }, - .len = SLEN(WARN_NM) + SLEN(WARN_VAL), - .nchunks = 2 - }; - - return __hdr_h2_add(resp, &wh); - } + TfwStr wh = { + .chunks = (TfwStr []){ + { .data = S_WARN, .len = SLEN(S_WARN) }, + { .data = S_V_WARN, .len = SLEN(S_V_WARN) } + }, + .len = SLEN(S_WARN) + SLEN(S_V_WARN), + .nchunks = 2 + }; - return 0; -#undef WARN_NM -#undef WARN_VAL + return tfw_hpack_encode(resp, &wh, TFW_H2_TRANS_EXPAND, false); } /* - * Split header in two parts: name and value, evicting ':' and OWS. + * Split header in two parts: name and value, evicting ':' and OWS. Return + * the resulting length of both parts. * - * NOTE: use this function with caution since it changes the underlying - * @TfwStr descriptors of @hdr; thus, this procedure should be used - * only in cases when the headers descriptors will not be needed any more - * (e.g. during message final transformation/adjusting, just before - * forwarding), or when the descriptors has been copied previously (e.g. - * via @tfw_strcpy_desc() function). + * NOTE: this function is intended for response processing only (during + * HTTP/1.1=>HTTP/2 transformation), since the response HTTP parser + * supports splitting the header name, colon, LWS, value and RWS into + * different chunks. */ -int -tfw_http_hdr_split(TfwStr *hdr, TfwStr *name_out, TfwStr *val_out) +unsigned long +tfw_http_hdr_split(TfwStr *hdr, TfwStr *name_out, TfwStr *val_out, bool inplace) { - bool name_found = false, val_found = false; - unsigned long tail, last_tail = 0, hdr_tail = 0; + unsigned long hdr_tail = 0; TfwStr *chunk, *end, *last_chunk = NULL; + bool name_found = false, val_found = false; BUG_ON(!TFW_STR_EMPTY(name_out) || !TFW_STR_EMPTY(val_out)); + if (WARN_ON_ONCE(TFW_STR_PLAIN(hdr))) + return 0; + + if (TFW_STR_EMPTY(hdr)) + return 0; + + if (!inplace) { + unsigned long off = 0; + /* + * During headers addition (or message expansion) the source + * @hdr must have the following chunk structure (without the + * OWS): + * + * { name [S_DLM] value1 [value2 [value3 ...]] }. + * + */ + *name_out = *hdr->chunks; + + chunk = TFW_STR_CHUNK(hdr, 1); + if (WARN_ON_ONCE(!chunk)) + return 0; + + if (chunk->len == SLEN(S_DLM) + && *(short *)chunk->data == *(short *)S_DLM) + { + off = SLEN(S_DLM); + chunk = TFW_STR_CHUNK(hdr, 2); + if (WARN_ON_ONCE(!chunk)) + return 0; + } + + val_out->chunks = chunk; + val_out->nchunks = hdr->chunks + hdr->nchunks - chunk; + val_out->len = hdr->len - name_out->len - off; + + return hdr->len - off; + } + name_out->chunks = hdr->chunks; TFW_STR_FOR_EACH_CHUNK(chunk, hdr, end) { - unsigned long idx; - if (!chunk->len) continue; if (!name_found) { - ++name_out->nchunks; - name_out->len += chunk->len; - if (chunk->data[chunk->len - 1] == ':') { - --name_out->len; - TFW_STR_LAST(name_out)->len -= 1; + if (chunk->data[0] == ':') { + WARN_ON_ONCE(chunk->len != 1); name_found = true; + } else { + ++name_out->nchunks; + name_out->len += chunk->len; } continue; } @@ -3654,8 +3898,7 @@ tfw_http_hdr_split(TfwStr *hdr, TfwStr *name_out, TfwStr *val_out) * value; thus, we can skip length of the entire (LWS) chunks. */ if (!val_found) { - if (unlikely(chunk->data[0] == ' ' - || chunk->data[0] == '\t')) + if (unlikely(chunk->flags & TFW_STR_OWS)) continue; val_out->chunks = chunk; @@ -3664,50 +3907,61 @@ tfw_http_hdr_split(TfwStr *hdr, TfwStr *name_out, TfwStr *val_out) val_out->len += chunk->len; - /* - * Skip OWS after the header value (RWS); accumulate the length - * in @tail for RWS cutting off (if this is not the end chunk, - * @tail will be reset). + /* + * Skip OWS after the header value (RWS) - they must be in + * separate chunks too. */ - tail = 0; - idx = chunk->len - 1; - while (chunk->data[idx] == ' ' - || chunk->data[idx] == '\t') - { - ++tail; - if (unlikely(!idx)) - break; - --idx; - } - - if (unlikely(tail == chunk->len)) { - hdr_tail += tail; + if (unlikely(chunk->flags & TFW_STR_OWS)) { + hdr_tail += chunk->len; } else { last_chunk = chunk; - last_tail = hdr_tail = tail; + hdr_tail = 0; } } - T_DBG3("%s: hdr_tail=%lu, val_out->len=%lu, last_tail=%lu," - " last_chunk->len=%lu, last_chunk->data='%.*s'\n", __func__, - hdr_tail, val_out->len, last_tail, last_chunk->len, - (int)last_chunk->len, last_chunk->data); + /* The header value is empty. */ + if (unlikely(!val_found)) + return name_out->len; if (WARN_ON_ONCE(!last_chunk)) - return -EINVAL; + return 0; - val_out->nchunks = chunk - val_out->chunks; + T_DBG3("%s: hdr_tail=%lu, val_out->len=%lu, last_chunk->len=%lu," + " last_chunk->data='%.*s'\n", __func__, hdr_tail, val_out->len, + last_chunk->len, (int)last_chunk->len, last_chunk->data); + + val_out->nchunks = last_chunk - val_out->chunks + 1; val_out->len -= hdr_tail; - last_chunk->len -= last_tail; - return 0; + return name_out->len + val_out->len; } -static int -tfw_h2_resp_add_loc_hdrs(TfwHttpResp *resp, const TfwHdrMods *h_mods) +unsigned long +tfw_h2_hdr_size(unsigned long n_len, unsigned long v_len, + unsigned short st_index) +{ + unsigned long size; + + if (st_index) { + size = tfw_hpack_int_size(st_index, 0xF); + } else { + size = 1; + size += tfw_hpack_int_size(n_len, 0x7F); + size += n_len; + } + size += tfw_hpack_int_size(v_len, 0x7F); + size += v_len; + + return size; +} + +int +tfw_h2_resp_add_loc_hdrs(TfwHttpResp *resp, const TfwHdrMods *h_mods, + bool cache) { unsigned int i; TfwHttpTransIter *mit = &resp->mit; + TfwH2TransOp op = cache ? TFW_H2_TRANS_EXPAND : TFW_H2_TRANS_ADD; if (!h_mods) return 0; @@ -3719,7 +3973,7 @@ tfw_h2_resp_add_loc_hdrs(TfwHttpResp *resp, const TfwHdrMods *h_mods) if (test_bit(i, mit->found) || !TFW_STR_CHUNK(desc->hdr, 1)) continue; - r = __hdr_h2_add(resp, desc->hdr); + r = tfw_hpack_encode(resp, desc->hdr, op, !cache); if (unlikely(r)) return r; } @@ -3751,16 +4005,18 @@ tfw_h2_resp_next_hdr(TfwHttpResp *resp, const TfwHdrMods *h_mods) for (i = mit->curr; i < map->count; ++i) { int k; + TfwStr *first; unsigned short hid = map->index[i].idx; unsigned short d_num = map->index[i].d_idx; TfwStr *tgt = &ht->tbl[hid]; - TfwStr *first = TFW_STR_CHUNK(tgt, 0); TfwHdrModsDesc *f_desc = NULL; const TfwStr *val; if (TFW_STR_DUP(tgt)) tgt = TFW_STR_CHUNK(tgt, d_num); + first = TFW_STR_CHUNK(tgt, 0); + if (WARN_ON_ONCE(!tgt || TFW_STR_EMPTY(tgt) || TFW_STR_DUP(tgt))) @@ -3778,8 +4034,7 @@ tfw_h2_resp_next_hdr(TfwHttpResp *resp, const TfwHdrMods *h_mods) if ((hid < TFW_HTTP_HDR_RAW && hid == desc->hid) || (hid >= TFW_HTTP_HDR_RAW - && !__hdr_name_cmp(tgt, - TFW_STR_CHUNK(desc->hdr, 0)))) + && !__hdr_name_cmp(tgt, desc->hdr))) { f_desc = desc; break; @@ -4044,13 +4299,16 @@ tfw_h2_error_resp(TfwHttpReq *req, int status, bool reply, bool attack, /* * If reply should be sent and this is not the attack case - we * can just send error response, leave the connection alive and - * drop request's corresponding stream; in this case stream will be - * switched into closed state locally (in @tfw_h2_resp_adjust_fwd() - * during error response sending, or below in @tfw_h2_stream_id_close(), - * if no error response is needed), and remotely - due to END_STREAM - * flag set in the last frame of error response; in case of attack - * we must close entire connection, and GOAWAY frame should be sent - * (RFC 7540 section 6.8) after error response. + * drop request's corresponding stream; in this case stream either + * is already in locally closed state (switched in + * @tfw_h2_stream_id_close() during failed proxy/internal response + * creation) or will be switched into locally closed state in + * @tfw_h2_send_resp() (or in @tfw_h2_stream_id_close() if no error + * response is needed) below; remotely (i.e. on client side) stream + * will be closed - due to END_STREAM flag set in the last frame of + * error response; in case of attack we must close entire connection, + * and GOAWAY frame should be sent (RFC 7540 section 6.8) after + * error response. */ if (reply) { tfw_h2_send_resp(req, status, 0); @@ -4267,7 +4525,8 @@ tfw_h2_resp_adjust_fwd(TfwHttpResp *resp) if (unlikely(r)) goto clean; - r = tfw_h2_pseudo_write(resp, TFW_H2_TRANS_SUB); + r = tfw_h2_resp_status_write(resp, resp->status, TFW_H2_TRANS_SUB, + false); if (unlikely(r)) goto clean; @@ -4289,7 +4548,7 @@ tfw_h2_resp_adjust_fwd(TfwHttpResp *resp) hdrs_end = true; } - r = tfw_hpack_encode(resp, &hdr, op); + r = tfw_hpack_encode(resp, &hdr, op, true); if (unlikely(r)) goto clean; @@ -4301,7 +4560,7 @@ tfw_h2_resp_adjust_fwd(TfwHttpResp *resp) * processed above and which have non-empty value (i.e. configured * not for deletion). */ - r = tfw_http_sess_resp_process(resp); + r = tfw_http_sess_resp_process(resp, false); if (unlikely(r)) goto clean; @@ -4309,22 +4568,17 @@ tfw_h2_resp_adjust_fwd(TfwHttpResp *resp) if (unlikely(r)) goto clean; - r = tfw_h2_set_stale_warn(resp); - if (unlikely(r)) - goto clean; - if (!test_bit(TFW_HTTP_B_HDR_DATE, resp->flags)) { - r = tfw_h2_add_hdr_date(resp); + r = tfw_h2_add_hdr_date(resp, TFW_H2_TRANS_ADD, false); if (unlikely(r)) goto clean; } - r = TFW_H2_MSG_HDR_ADD(resp, "server", TFW_NAME "/" TFW_VERSION, - TFW_HTTP_HDR_SERVER, 54); + r = TFW_H2_MSG_HDR_ADD(resp, "server", TFW_SERVER, 54); if (unlikely(r)) goto clean; - r = tfw_h2_resp_add_loc_hdrs(resp, h_mods); + r = tfw_h2_resp_add_loc_hdrs(resp, h_mods, false); if (unlikely(r)) goto clean; @@ -4367,18 +4621,11 @@ tfw_http_req_cache_service(TfwHttpResp *resp) WARN_ON_ONCE(!list_empty(&req->fwd_list)); WARN_ON_ONCE(!list_empty(&req->nip_list)); - if (TFW_MSG_H2(req)) { - tfw_h2_resp_adjust_fwd(resp); - } else { - if (tfw_http_adjust_resp(resp)) { - tfw_http_conn_msg_free((TfwHttpMsg *)resp); - tfw_http_send_resp(req, 500, "response dropped:" - " processing error"); - TFW_INC_STAT_BH(clnt.msgs_otherr); - return; - } + if (TFW_MSG_H2(req)) + tfw_h2_resp_fwd(resp); + else tfw_http_resp_fwd(resp); - } + TFW_INC_STAT_BH(clnt.msgs_fromcache); } diff --git a/tempesta_fw/http.h b/tempesta_fw/http.h index 632333e762..b17b0e4633 100644 --- a/tempesta_fw/http.h +++ b/tempesta_fw/http.h @@ -284,8 +284,6 @@ enum { TFW_HTTP_B_HDR_DATE, /* Response has header 'Last-Modified:'. */ TFW_HTTP_B_HDR_LMODIFIED, - /* Response is stale, but pass with a warning. */ - TFW_HTTP_B_RESP_STALE, /* Response is fully processed and ready to be forwarded to the client. */ TFW_HTTP_B_RESP_READY, @@ -508,6 +506,8 @@ typedef struct { * Iterator for message HTTP/2 transformation process. * * @map - indirection map for tracking headers order in skb; + * @start_off - initial offset during copying response data into + * skb (for subsequent insertion of HTTP/2 frame header); * @curr - current header index in the @map; * @next - operation (with necessary attributes) which should be executed * with next header; @@ -520,6 +520,7 @@ typedef struct { */ typedef struct { TfwHttpHdrMap *map; + unsigned int start_off; unsigned int curr; TfwNextHdrOp next; DECLARE_BITMAP (found, TFW_USRHDRS_ARRAY_SZ); @@ -614,6 +615,33 @@ tfw_http_resp_code_range(const int n) return n <= HTTP_CODE_MAX && n >= HTTP_CODE_MIN; } +/* + * Static index determination for response ':status' pseudo-header (see RFC + * 7541 Appendix A for details). + */ +static inline unsigned short +tfw_h2_pseudo_index(unsigned short status) +{ + switch (status) { + case 200: + return 8; + case 204: + return 9; + case 206: + return 10; + case 304: + return 11; + case 400: + return 12; + case 404: + return 13; + case 500: + return 14; + default: + return 0; + } +} + typedef void (*tfw_http_cache_cb_t)(TfwHttpMsg *); /* External HTTP functions. */ @@ -626,20 +654,35 @@ void tfw_http_resp_fwd(TfwHttpResp *resp); void tfw_http_resp_build_error(TfwHttpReq *req); int tfw_cfgop_parse_http_status(const char *status, int *out); void tfw_http_hm_srv_send(TfwServer *srv, char *data, unsigned long len); +int tfw_http_set_loc_hdrs(TfwHttpMsg *hm, TfwHttpReq *req, bool cache); +int tfw_http_expand_stale_warn(TfwHttpResp *resp); +int tfw_http_expand_hdr_date(TfwHttpResp *resp); +int tfw_http_expand_hbh(TfwHttpResp *resp, unsigned short status); +void tfw_h2_resp_fwd(TfwHttpResp *resp); int tfw_h2_hdr_map(TfwHttpResp *resp, const TfwStr *hdr, unsigned int id); - +int tfw_h2_add_hdr_date(TfwHttpResp *resp, TfwH2TransOp op, bool cache); +int tfw_h2_set_stale_warn(TfwHttpResp *resp); +int tfw_h2_resp_add_loc_hdrs(TfwHttpResp *resp, const TfwHdrMods *h_mods, + bool cache); +int tfw_h2_resp_status_write(TfwHttpResp *resp, unsigned short status, + TfwH2TransOp op, bool cache); /* * Functions to send an HTTP error response to a client. */ -int tfw_http_prep_redirect(TfwHttpMsg *resp, unsigned short status, - TfwStr *rmark, TfwStr *cookie, TfwStr *body); -int tfw_http_prep_304(TfwHttpMsg *resp, TfwHttpReq *req, TfwMsgIter *msg_it, - size_t hdrs_size); +int tfw_h2_prep_redirect(TfwHttpResp *resp, unsigned short status, + TfwStr *rmark, TfwStr *cookie, TfwStr *body); +int tfw_h1_prep_redirect(TfwHttpResp *resp, unsigned short status, + TfwStr *rmark, TfwStr *cookie, TfwStr *body); +int tfw_http_prep_304(TfwHttpReq *req, struct sk_buff **skb_head, + TfwMsgIter *it); void tfw_http_conn_msg_free(TfwHttpMsg *hm); void tfw_http_send_resp(TfwHttpReq *req, int status, const char *reason); /* Helper functions */ char *tfw_http_msg_body_dup(const char *filename, size_t *len); -int tfw_http_hdr_split(TfwStr *hdr, TfwStr *name_out, TfwStr *val_out); +unsigned long tfw_http_hdr_split(TfwStr *hdr, TfwStr *name_out, TfwStr *val_out, + bool inplace); +unsigned long tfw_h2_hdr_size(unsigned long n_len, unsigned long v_len, + unsigned short st_index); #endif /* __TFW_HTTP_H__ */ diff --git a/tempesta_fw/http_frame.c b/tempesta_fw/http_frame.c index 32dcbb2f67..ec344de8d4 100644 --- a/tempesta_fw/http_frame.c +++ b/tempesta_fw/http_frame.c @@ -694,11 +694,30 @@ tfw_h2_stream_close(TfwH2Ctx *ctx, unsigned int id, TfwStream **stream, } /* - * Get stream ID for upper layer to prepare and send frame (of type specified - * in @type and with flags set in @flags) with response to client. This - * procedure also unlinks request from corresponding stream (if linked) and - * moves the stream to the queue of closed streams (if it is not contained - * there yet). + * Get stream ID for upper layer to create frames info. + */ +unsigned int +tfw_h2_stream_id(TfwHttpReq *req) +{ + unsigned int id = 0; + TfwH2Ctx *ctx = tfw_h2_context(req->conn); + + spin_lock(&ctx->lock); + + if (req->stream) + id = req->stream->id; + + spin_unlock(&ctx->lock); + + return id; +} + +/* + * Get stream ID for upper layer to prepare and send frame with response to + * client, and process stream FSM for the frame (of type specified in @type + * and with flags set in @flags). This procedure also unlinks request from + * corresponding stream (if linked) and moves the stream to the queue of + * closed streams (if it is not contained there yet). */ unsigned int tfw_h2_stream_id_close(TfwHttpReq *req, unsigned char type, @@ -722,6 +741,7 @@ tfw_h2_stream_id_close(TfwHttpReq *req, unsigned char type, id = stream->id; } + req->stream = NULL; stream->msg = NULL; __tfw_h2_stream_add_closed(&ctx->hclosed_streams, stream); diff --git a/tempesta_fw/http_frame.h b/tempesta_fw/http_frame.h index b44b1ecb65..ea83dde2b5 100644 --- a/tempesta_fw/http_frame.h +++ b/tempesta_fw/http_frame.h @@ -206,6 +206,7 @@ int tfw_h2_context_init(TfwH2Ctx *ctx); void tfw_h2_context_clear(TfwH2Ctx *ctx); int tfw_h2_frame_process(void *c, TfwFsmData *data); void tfw_h2_conn_streams_cleanup(TfwH2Ctx *ctx); +unsigned int tfw_h2_stream_id(TfwHttpReq *req); unsigned int tfw_h2_stream_id_close(TfwHttpReq *req, unsigned char type, unsigned char flags); void tfw_h2_conn_terminate_close(TfwH2Ctx *ctx, TfwH2Err err_code, bool close); diff --git a/tempesta_fw/http_msg.c b/tempesta_fw/http_msg.c index 2fd4b548ac..bc02a4bc62 100644 --- a/tempesta_fw/http_msg.c +++ b/tempesta_fw/http_msg.c @@ -166,8 +166,12 @@ tfw_http_msg_req_spec_hid(const TfwStr *hdr) } /** - * Fills @val with second part of special HTTP header containing the header - * value. + * Fills @val with second part of special HTTP/1.1 header containing the + * header value. + * + * TODO: with the current HTTP-parser implementation (parsing header name, + * colon, LWS and value into different chunks) this procedure can be + * simplified to avoid the usage of predefined header arrays. */ void __http_msg_hdr_val(TfwStr *hdr, unsigned id, TfwStr *val, bool client) @@ -260,35 +264,6 @@ __http_msg_hdr_val(TfwStr *hdr, unsigned id, TfwStr *val, bool client) } EXPORT_SYMBOL(__http_msg_hdr_val); -void -__h2_msg_hdr_name(TfwStr *hdr, TfwStr *out_name) -{ - const TfwStr *c, *end; - - if (unlikely(TFW_STR_EMPTY(hdr))) { - TFW_STR_INIT(out_name); - return; - } - - BUG_ON(TFW_STR_DUP(hdr)); - BUG_ON(TFW_STR_EMPTY(hdr)); - - *out_name = *hdr; - - if (unlikely(TFW_STR_PLAIN(hdr))) { - WARN_ON_ONCE(hdr->flags & TFW_STR_HDR_VALUE); - return; - } - - TFW_STR_FOR_EACH_CHUNK(c, hdr, end) { - if (c->flags & TFW_STR_HDR_VALUE) { - out_name->len -= c->len; - out_name->nchunks--; - } - } -} -EXPORT_SYMBOL(__h2_msg_hdr_name); - void __h2_msg_hdr_val(TfwStr *hdr, TfwStr *out_val) { @@ -367,9 +342,19 @@ tfw_http_msg_hdr_lookup(TfwHttpMsg *hm, const TfwStr *hdr) * Certain header fields are strictly singular and may not be repeated in * an HTTP message. Duplicate of a singular header fields is a bug worth * blocking the whole HTTP message. + * + * TODO: with the current HTTP-parser implementation (parsing header name, + * colon, LWS and value into different chunks) we can avoid slow string + * matcher, which is used in @tfw_http_msg_hdr_lookup(), and can compare + * strings just by chunks (including searching the stop character) for both + * HTTP/2 and HTTP/1.1 formatted headers (see @__hdr_name_cmp() below). + * Thus, @__h1_hdr_lookup() and @tfw_http_msg_hdr_lookup() procedures should + * be unified to @__hdr_name_cmp() and @__http_hdr_lookup() in order to + * substitute current mess of multiple partially duplicated procedures with + * one simple interface. */ static inline unsigned int -__hdr_lookup(TfwHttpMsg *hm, const TfwStr *hdr) +__h1_hdr_lookup(TfwHttpMsg *hm, const TfwStr *hdr) { unsigned int id = tfw_http_msg_hdr_lookup(hm, hdr); @@ -380,33 +365,36 @@ __hdr_lookup(TfwHttpMsg *hm, const TfwStr *hdr) } /** - * Special procedure comparing specified name against the header in HTTP/2 - * or HTTP/1.1 format. + * Special procedure comparing the name or HPACK static index of @cmp_hdr (can + * be in HTTP/2 or HTTP/1.1 format) against the header @hdr which also can be + * in HTTP/2 or HTTP/1.1 format. */ int -__hdr_name_cmp(const TfwStr *hdr, const TfwStr *name) +__hdr_name_cmp(const TfwStr *hdr, const TfwStr *cmp_hdr) { long n; int i1, i2, off1, off2; const TfwStr *c1, *c2; BUG_ON(hdr->flags & TFW_STR_DUPLICATE); - BUG_ON(!name->len); + BUG_ON(!cmp_hdr->len); + + if (cmp_hdr->hpack_idx && cmp_hdr->hpack_idx == hdr->hpack_idx) + return 0; if (unlikely(!hdr->len)) - return -name->len; + return 1; i1 = i2 = 0; off1 = off2 = 0; - n = min(hdr->len, name->len); + n = min(hdr->len, cmp_hdr->len); c1 = TFW_STR_CHUNK(hdr, 0); - c2 = TFW_STR_CHUNK(name, 0); + c2 = TFW_STR_CHUNK(cmp_hdr, 0); while (n) { int cn = min(c1->len - off1, c2->len - off2); - int r = tfw_cstricmp(c1->data + off1, - c2->data + off2, cn); - if (r) - return r; + + if (tfw_cstricmp(c1->data + off1, c2->data + off2, cn)) + return 1; n -= cn; if (cn == c1->len - off1) { @@ -419,36 +407,44 @@ __hdr_name_cmp(const TfwStr *hdr, const TfwStr *name) if (cn == c2->len - off2) { off2 = 0; ++i2; - c2 = TFW_STR_CHUNK(name, i2); + c2 = TFW_STR_CHUNK(cmp_hdr, i2); } else { off2 += cn; } - BUG_ON(n && (!c1 || !c2)); - } - /* Only name is contained in the header. */ - if (hdr->len == name->len) - return 0; + BUG_ON(n && (!c1 || !c2)); - if (hdr->len > name->len) { /* - * If the header is of HTTP/2 format, the end of name must match - * the end of the chunk, and the following value must have - * appropriate flag. + * Regardless of the header format (HTTP/2 or HTTP/1.1), the end + * of the name must match the end of the chunk, and the following + * chunk must contain value with appropriate flag (or it must + * contain just a single colon in case of HTTP/1.1-header). */ - if (!off1 - && (c1->flags & TFW_STR_HDR_VALUE) - && !(TFW_STR_CHUNK(hdr, i1 - 1)->flags & TFW_STR_HDR_VALUE)) - return 0; - /* - * If this is the HTTP/1.1-format header, the value must begin - * after the colon. - */ - if (*(c1->data + off1) == ':') - return 0; + if (!off2) { + const TfwStr *prev_c1; + /* + * If @c2 or @c1 is NULL, then only name is contained in + * the @cmp_hdr or @hdr respectively. + */ + if (c2 + && !(c2->flags & TFW_STR_HDR_VALUE) + && *c2->data != ':') + continue; + + prev_c1 = TFW_STR_CHUNK(hdr, i1 - 1); + + if (!off1 + && !(prev_c1->flags & TFW_STR_HDR_VALUE) + && (!c1 + || c1->flags & TFW_STR_HDR_VALUE + || *c1->data == ':')) + return 0; + + return 1; + } } - return (long)hdr->len - (long)name->len; + return 1; } /** @@ -457,7 +453,7 @@ __hdr_name_cmp(const TfwStr *hdr, const TfwStr *name) * headers in HTTP/2 or HTTP/1.1 format. */ int -__h2_hdr_lookup(TfwHttpMsg *hm, const TfwStr *h_name) +__http_hdr_lookup(TfwHttpMsg *hm, const TfwStr *hdr) { unsigned int id; TfwHttpHdrTbl *ht = hm->h_tbl; @@ -470,7 +466,7 @@ __h2_hdr_lookup(TfwHttpMsg *hm, const TfwStr *h_name) */ if (h->flags & TFW_STR_DUPLICATE) h = TFW_STR_CHUNK(h, 0); - if (!__hdr_name_cmp(h, h_name)) + if (!__hdr_name_cmp(h, hdr)) break; } @@ -548,15 +544,7 @@ tfw_http_msg_hdr_close(TfwHttpMsg *hm) * Both the headers, the new one and existing one, can already be * compound. */ - if (TFW_MSG_H2(hm)) { - TfwStr h_name; - - __h2_msg_hdr_name(&parser->hdr, &h_name); - id = __h2_hdr_lookup(hm, &h_name); - } - else { - id = __hdr_lookup(hm, &parser->hdr); - } + id = __http_hdr_lookup(hm, &parser->hdr); /* Allocate some more room if not enough to store the header. */ if (unlikely(id == ht->size)) { @@ -698,12 +686,6 @@ __hdr_add(TfwHttpMsg *hm, const TfwStr *hdr, unsigned int hid) return 0; } -int -__hdr_h2_add(TfwHttpResp *resp, TfwStr *hdr) -{ - return tfw_hpack_encode(resp, hdr, TFW_H2_TRANS_ADD); -} - /** * Expand @orig_hdr by appending or replacing with the @hdr. * (CRLF is not accounted in TfwStr representation of HTTP headers). @@ -859,7 +841,7 @@ tfw_http_msg_hdr_xfrm_str(TfwHttpMsg *hm, const TfwStr *hdr, unsigned int hid, /* Not found, nothing to delete. */ return 0; } else { - hid = __hdr_lookup(hm, hdr); + hid = __h1_hdr_lookup(hm, hdr); if (hid == ht->off && !s_val) /* Not found, nothing to delete. */ return 0; @@ -1300,173 +1282,9 @@ __tfw_http_msg_alloc(int type, bool full) return hm; } -/** - * Determination length of the header's real part (for details see comment - * for @tfw_h2_msg_hdr_write() below) to store it in the encoder dynamic - * index. - */ -unsigned long -tfw_h2_msg_hdr_length(const TfwStr *hdr, unsigned long *name_len, - unsigned long *val_off, unsigned long *val_len, - TfwH2TransOp op) -{ - const TfwStr *chunk, *end; - unsigned long tail, hdr_tail = 0, hdr_len = 0; - bool name_found = false, val_found = false; - - *name_len = *val_off = *val_len = 0; - - if (op != TFW_H2_TRANS_INPLACE) { - /* - * During headers addition (or message expansion) the the source - * @hdr must have the following chunk structure (without the - * OWS): - * - * { name [S_DLM] value1 [value2 [value3 ...]] }. - * - */ - chunk = TFW_STR_CHUNK(hdr, 1); - if (WARN_ON_ONCE(!chunk)) - return 0; - - if (chunk->len == SLEN(S_DLM) - && *(short *)chunk->data == *(short *)S_DLM) - { - *val_off = SLEN(S_DLM); - } - - hdr_len = hdr->len; - *name_len = TFW_STR_CHUNK(hdr, 0)->len; - *val_len = hdr_len - *name_len - *val_off; - - return hdr_len - *val_off; - } - - TFW_STR_FOR_EACH_CHUNK(chunk, hdr, end) { - unsigned long idx; - - if (!chunk->len) - continue; - - hdr_len += chunk->len; - if (!name_found) { - *name_len += chunk->len; - if (chunk->data[chunk->len - 1] == ':') { - --*name_len; - name_found = true; - } - continue; - } - /* - * Skip OWS before the header value (LWS) during HTTP/2 header's - * real length calculation. LWS is always in the separate chunks - * between the name and value; thus, we can skip length of the - * entire (LWS) chunks. - */ - if (!val_found) { - if (unlikely(chunk->data[0] == ' ' - || chunk->data[0] == '\t')) - { - *val_off += chunk->len; - continue; - } - /* - * The colon must not be included into HTTP/2 header, - * thus, it should be counted in the value offset. - */ - ++*val_off; - val_found = true; - } - /* - * Skip OWS after the header value (RWS); accumulate the length - * in @tail for RWS cutting off (if this is not the end chunk, - * @tail will be reset). - */ - tail = 0; - idx = chunk->len - 1; - while (chunk->data[idx] == ' ' - || chunk->data[idx] == '\t') - { - ++tail; - if (unlikely(!idx)) - break; - --idx; - } - - if (unlikely(tail == chunk->len)) - hdr_tail += tail; - else - hdr_tail = tail; - } - - WARN_ON_ONCE(!name_found); - - *val_len = hdr_len - *name_len - *val_off - hdr_tail; - - T_DBG3("%s: name_len=%lu, val_off=%lu, val_len=%lu, hdr_tail=%lu," - " hdr_len=%lu\n", __func__, *name_len, *val_off, *val_len, - hdr_tail, hdr_len); - - return *name_len + *val_len; -} - -/** - * Copy the real part of header (i.e. the header in HTTP/2 form - without name - * colon and OWS) into @out_buf from @hdr; @nm_len is the real length of header - * name, @val_len - the real length of header value, and @val_off - the offset - * between header name and value (i.e. the part occupied by colon and OWS); OWS - * in the end of header's value are also skipped and will not be included into - * header's copied part. Note that the size of prepared @out_buf must be not - * less than sum of @nm_len and @val_len. - */ -void -tfw_h2_msg_hdr_write(const TfwStr *hdr, unsigned long nm_len, - unsigned long val_off, unsigned long val_len, - char *out_buf) -{ - const TfwStr *c, *end; - - T_DBG3("%s: enter, nm_len=%lu, val_off=%lu, val_len=%lu\n", __func__, - nm_len, val_off, val_len); - - BUG_ON(!nm_len); - TFW_STR_FOR_EACH_CHUNK(c, hdr, end) { - unsigned long len; - - if (!c->len) - continue; - - len = 0; - if (nm_len) { - len = min(nm_len, c->len); - nm_len -= len; - } - - if (!nm_len) { - if (val_off) { - WARN_ON_ONCE(val_off < c->len - len); - val_off -= c->len - len; - } - else if (val_len && !len) { - len = min(val_len, c->len); - val_len -= len; - } - } - - if (!len) - continue; - - T_DBG3("%s: len=%lu, c->data='%.*s'\n", __func__, len, (int)len, - c->data); - - memcpy_fast(out_buf, c->data, len); - out_buf += len; - } -} - int tfw_http_msg_expand_data(TfwMsgIter *it, struct sk_buff **skb_head, - const TfwStr *src) + const TfwStr *src, unsigned int *start_off) { const TfwStr *c, *end; @@ -1477,10 +1295,21 @@ tfw_http_msg_expand_data(TfwMsgIter *it, struct sk_buff **skb_head, if (!it->skb) { if (!(it->skb = ss_skb_alloc(SKB_MAX_HEADER))) return -ENOMEM; + /* + * Expanding skb is always used for TLS client + * connections. + */ + skb_shinfo(it->skb)->tx_flags |= SKBTX_SHARED_FRAG; ss_skb_queue_tail(skb_head, it->skb); it->frag = -1; - if (!it->skb_head) + if (!it->skb_head) { it->skb_head = *skb_head; + + if (start_off && *start_off) { + skb_put(it->skb_head, *start_off); + *start_off = 0; + } + } T_DBG3("message expanded by new skb [%p]\n", it->skb); } diff --git a/tempesta_fw/http_msg.h b/tempesta_fw/http_msg.h index 92b00d7c81..24b93c2ebc 100644 --- a/tempesta_fw/http_msg.h +++ b/tempesta_fw/http_msg.h @@ -27,9 +27,22 @@ #define S_DLM ": " #define S_SET_COOKIE "set-cookie" #define S_F_SET_COOKIE S_SET_COOKIE S_DLM +#define S_LOCATION "location" +#define S_F_LOCATION S_LOCATION S_DLM +#define S_VIA "via" +#define S_F_VIA S_VIA S_DLM +#define S_VIA_H2_PROTO "2.0 " +#define S_VERSION11 "HTTP/1.1" +#define S_0 S_VERSION11 " " #define SLEN(s) (sizeof(s) - 1) +/* + * The size of the buffer to store the value for ':status' pseudo-header + * of HTTP/2-response. + */ +#define H2_STAT_VAL_LEN 3 + TfwStr *tfw_http_msg_make_hdr(TfwPool *pool, const char *name, const char *val); unsigned int tfw_http_msg_resp_spec_hid(const TfwStr *hdr); unsigned int tfw_http_msg_req_spec_hid(const TfwStr *hdr); @@ -50,8 +63,6 @@ __tfw_http_msg_set_str_data(TfwStr *str, void *data, struct sk_buff *skb) __tfw_http_msg_set_str_data(str, data, \ ss_skb_peek_tail(&hm->msg.skb_head)) -int __hdr_h2_add(TfwHttpResp *resp, TfwStr *hdr); -void __h2_msg_hdr_name(TfwStr *hdr, TfwStr *out_name); void __h2_msg_hdr_val(TfwStr *hdr, TfwStr *out_val); void __http_msg_hdr_val(TfwStr *hdr, unsigned id, TfwStr *val, bool client); @@ -125,7 +136,7 @@ tfw_h2_msg_transform_setup(TfwHttpTransIter *mit, struct sk_buff *skb, static inline int tfw_h2_msg_hdr_add(TfwHttpResp *resp, char *name, size_t nlen, char *val, - size_t vlen, unsigned int hid, unsigned short idx) + size_t vlen, unsigned short idx) { TfwStr hdr = { .chunks = (TfwStr []){ @@ -137,7 +148,7 @@ tfw_h2_msg_hdr_add(TfwHttpResp *resp, char *name, size_t nlen, char *val, .hpack_idx = idx }; - return __hdr_h2_add(resp, &hdr); + return tfw_hpack_encode(resp, &hdr, TFW_H2_TRANS_ADD, true); } int __tfw_http_msg_add_str_data(TfwHttpMsg *hm, TfwStr *str, void *data, @@ -173,21 +184,14 @@ int tfw_http_msg_hdr_close(TfwHttpMsg *hm); int tfw_http_msg_grow_hdr_tbl(TfwHttpMsg *hm); void tfw_http_msg_free(TfwHttpMsg *m); int tfw_http_msg_expand_data(TfwMsgIter *it, struct sk_buff **skb_head, - const TfwStr *src); -int __hdr_name_cmp(const TfwStr *hdr, const TfwStr *name); -int __h2_hdr_lookup(TfwHttpMsg *hm, const TfwStr *h_name); -unsigned long tfw_h2_msg_hdr_length(const TfwStr *hdr, unsigned long *name_len, - unsigned long *val_off, - unsigned long *val_len, - TfwH2TransOp op); -void tfw_h2_msg_hdr_write(const TfwStr *hdr, unsigned long nm_len, - unsigned long val_off, unsigned long val_len, - char *out_buf); + const TfwStr *src, unsigned int *start_off); +int __hdr_name_cmp(const TfwStr *hdr, const TfwStr *cmp_hdr); +int __http_hdr_lookup(TfwHttpMsg *hm, const TfwStr *hdr); int tfw_h2_msg_rewrite_data(TfwHttpTransIter *mit, const TfwStr *str, const char *stop); -#define TFW_H2_MSG_HDR_ADD(hm, name, val, hid, idx) \ +#define TFW_H2_MSG_HDR_ADD(hm, name, val, idx) \ tfw_h2_msg_hdr_add(hm, name, sizeof(name) - 1, val, \ - sizeof(val) - 1, hid, idx) + sizeof(val) - 1, idx) #endif /* __TFW_HTTP_MSG_H__ */ diff --git a/tempesta_fw/http_parser.c b/tempesta_fw/http_parser.c index e0a3e8e3d1..10c96921b5 100644 --- a/tempesta_fw/http_parser.c +++ b/tempesta_fw/http_parser.c @@ -83,6 +83,15 @@ do { \ (field)->flags |= TFW_STR_COMPLETE; \ } while (0) +#define __msg_field_chunk_flags(field, flag) \ +do { \ + T_DBG3("parser: add chunk flags: %u\n", flag); \ + TFW_STR_CURR(field)->flags |= flag; \ +} while (0) + +#define __msg_chunk_flags(flag) \ + __msg_field_chunk_flags(&msg->stream->parser.hdr, flag) + #define __msg_hdr_chunk_fixup(data, len) \ tfw_http_msg_add_str_data(msg, &msg->stream->parser.hdr, data, len) @@ -183,7 +192,7 @@ do { \ /* The same as __FSM_MOVE_n(), but exactly for jumps w/o data moving. */ #define __FSM_JMP(to) do { goto to; } while (0) -#define __FSM_MATCH_MOVE_fixup_pos(alphabet, to, field, fixup_pos) \ +#define __FSM_MATCH_MOVE_fixup_pos(alphabet, to, field, flag, fixup_pos) \ do { \ __fsm_n = __data_remain(p); \ __fsm_sz = tfw_match_##alphabet(p, __fsm_n); \ @@ -194,20 +203,21 @@ do { \ __msg_field_fixup_pos(field, p, __fsm_sz); \ else \ __msg_field_fixup(field, data + len); \ + __msg_field_chunk_flags(field, flag); \ parser->state = &&to; \ p += __fsm_sz; \ __FSM_EXIT(TFW_POSTPONE); \ } \ } while (0) -#define __FSM_MATCH_MOVE_f(alphabet, to, field) \ - __FSM_MATCH_MOVE_fixup_pos(alphabet, to, field, false) +#define __FSM_MATCH_MOVE_f(alphabet, to, field, flag) \ + __FSM_MATCH_MOVE_fixup_pos(alphabet, to, field, flag, false) -#define __FSM_MATCH_MOVE_pos_f(alphabet, to, field) \ - __FSM_MATCH_MOVE_fixup_pos(alphabet, to, field, true) +#define __FSM_MATCH_MOVE_pos_f(alphabet, to, field, flag) \ + __FSM_MATCH_MOVE_fixup_pos(alphabet, to, field, flag, true) -#define __FSM_MATCH_MOVE(alphabet, to) \ - __FSM_MATCH_MOVE_f(alphabet, to, &msg->stream->parser.hdr) +#define __FSM_MATCH_MOVE(alphabet, to, flag) \ + __FSM_MATCH_MOVE_f(alphabet, to, &msg->stream->parser.hdr, flag) #define __FSM_MATCH_MOVE_nofixup(alphabet, to) \ do { \ @@ -220,19 +230,27 @@ do { \ } \ } while (0) +#define __FSM_MOVE_hdr_fixup(to, n) \ +do { \ + __msg_hdr_chunk_fixup(p, n); \ + p += n; \ + if (unlikely(__data_off(p) >= len)) { \ + parser->state = &&to; \ + __FSM_EXIT(TFW_POSTPONE); \ + } \ + goto to; \ +} while (0) + /* * __FSM_I_* macros are intended to help with parsing of message * header values. That is done with separate, nested, or interior * FSMs, and so _I_ in the name means "interior" FSM. */ #define __FSM_I_field_chunk_flags(field, flag) \ -do { \ - T_DBG3("parser: add chunk flags: %u\n", flag); \ - TFW_STR_CURR(field)->flags |= flag; \ -} while (0) + __msg_field_chunk_flags(field, flag) #define __FSM_I_chunk_flags(flag) \ - __FSM_I_field_chunk_flags(&msg->stream->parser.hdr, flag) + __msg_chunk_flags(flag) #define __FSM_I_MOVE_BY_REF_n(to, n) \ do { \ @@ -349,12 +367,16 @@ __FSM_STATE(st, cold) { \ __FSM_JMP(RGen_HdrOtherN); \ } -/* As above, but reads OWS through transitional state. */ +/* + * As above, but reads OWS through transitional state. Note, that header + * name, colon, LWS and value are stored in different chunks. + */ #define __FSM_TX_AF_OWS(st, st_next) \ __FSM_STATE(st, cold) { \ if (likely(c == ':')) { \ + __msg_hdr_chunk_fixup(data, __data_off(p)); \ parser->_i_st = &&st_next; \ - __FSM_MOVE(RGen_LWS); \ + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); \ } \ /* It should be checked in st_fallback if `c` is allowed */ \ __FSM_JMP(RGen_HdrOtherN); \ @@ -364,9 +386,10 @@ __FSM_STATE(st, cold) { \ #define __FSM_TX_AF_OWS_HP(st, st_next, hp_idx) \ __FSM_STATE(st, cold) { \ if (likely(c == ':')) { \ + __msg_hdr_chunk_fixup(data, __data_off(p)); \ parser->_i_st = &&st_next; \ __msg_hdr_set_hpack_index(hp_idx); \ - __FSM_MOVE(RGen_LWS); \ + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); \ } \ /* It should be checked in st_fallback if `c` is allowed */ \ __FSM_JMP(RGen_HdrOtherN); \ @@ -832,6 +855,9 @@ mark_trailer_hdr(TfwHttpMsg *hm, TfwStr *hdr) return CSTR_POSTPONE; \ } +#define TRY_STR_fixup(str, curr_st, next_st) \ + TRY_STR_LAMBDA_fixup(str, &parser->hdr, { }, curr_st, next_st) + /* * Headers EOL processing. Allow only LF and CRLF as a newline delimiters. * @@ -1004,10 +1030,12 @@ __FSM_STATE(st_curr) { \ */ #define RGEN_HDR_OTHER() \ __FSM_STATE(RGen_HdrOtherN) { \ - __FSM_MATCH_MOVE(token, RGen_HdrOtherN); \ + __FSM_MATCH_MOVE(token, RGen_HdrOtherN, 0); \ if (likely(*(p + __fsm_sz) == ':')) { \ + __msg_hdr_chunk_fixup(data, __data_off(p + __fsm_sz)); \ parser->_i_st = &&RGen_HdrOtherV; \ - __FSM_MOVE_n(RGen_LWS, __fsm_sz + 1); \ + p += __fsm_sz; \ + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); \ } \ TFW_PARSER_BLOCK(RGen_HdrOtherN); \ } \ @@ -1017,7 +1045,7 @@ __FSM_STATE(RGen_HdrOtherV) { \ * so pass ctext and VCHAR. \ */ \ __FSM_MATCH_MOVE_pos_f(ctext_vchar, RGen_HdrOtherV, \ - &msg->stream->parser.hdr); \ + &msg->stream->parser.hdr, 0); \ if (!IS_CRLF(*(p + __fsm_sz))) \ TFW_PARSER_BLOCK(RGen_HdrOtherV); \ __msg_hdr_chunk_fixup(p, __fsm_sz); \ @@ -1209,21 +1237,22 @@ __FSM_STATE(RGen_BodyCR, __VA_ARGS__) { \ */ #define RGEN_OWS() \ __FSM_STATE(RGen_LWS, hot) { \ - /* Store header name, LWS and value in different chunks. */ \ - __msg_hdr_chunk_fixup(data, p - data); \ __fsm_sz = __data_remain(p); \ __fsm_n = parse_ows(p, __fsm_sz); \ T_DBG3("parse LWS: __fsm_n=%d, __fsm_sz=%lu, len=%lu," \ " off=%lu\n", __fsm_n, __fsm_sz, len, __data_off(p)); \ if (__fsm_n == CSTR_POSTPONE) { \ __msg_hdr_chunk_fixup(p, __fsm_sz); \ + __msg_chunk_flags(TFW_STR_OWS); \ p += __fsm_sz; \ parser->state = &&RGen_LWS; \ __FSM_EXIT(TFW_POSTPONE); \ } \ BUG_ON(__fsm_n < 0); \ - if (__fsm_n) \ + if (__fsm_n) { \ __msg_hdr_chunk_fixup(p, __fsm_n); \ + __msg_chunk_flags(TFW_STR_OWS); \ + } \ parser->state = parser->_i_st; \ parser->_i_st = NULL; \ p += __fsm_n; \ @@ -1820,7 +1849,8 @@ __parse_transfer_encoding(TfwHttpMsg *hm, unsigned char *data, size_t len, * to a message body (i.e., chunking an already * chunked message is not allowed). RFC 7230 3.3.1. */ - TRY_STR("chunked", I_TransEncodTok, I_TransEncodChunked); + TRY_STR_fixup(&TFW_STR_STRING("chunked"), I_TransEncodTok, + I_TransEncodChunked); TRY_STR_INIT(); __FSM_I_JMP(I_TransEncodOther); } @@ -1835,7 +1865,6 @@ __parse_transfer_encoding(TfwHttpMsg *hm, unsigned char *data, size_t len, __FSM_I_JMP(I_TransEncodOther); } - /* * RFC 7230 3.3.1: * If any transfer coding @@ -1850,10 +1879,13 @@ __parse_transfer_encoding(TfwHttpMsg *hm, unsigned char *data, size_t len, * compress; */ __FSM_STATE(I_TransEncodOther) { - __FSM_I_MATCH_MOVE(token, I_TransEncodOther); + __FSM_I_MATCH_MOVE_fixup(token, I_TransEncodOther, 0); c = *(p + __fsm_sz); - if (IS_WS(c) || c == ',') - __FSM_I_MOVE_n(I_EoT, __fsm_sz + 1); + if (IS_WS(c) || c == ',') { + __msg_hdr_chunk_fixup(p, __fsm_sz); + p += __fsm_sz; + __FSM_I_JMP(I_EoT); + } if (IS_CRLF(c)) { if (unlikely(test_bit(TFW_HTTP_B_CHUNKED, msg->flags))) { @@ -1863,6 +1895,7 @@ __parse_transfer_encoding(TfwHttpMsg *hm, unsigned char *data, size_t len, __set_bit(TFW_HTTP_B_CHUNKED_APPLIED, msg->flags); } + __msg_hdr_chunk_fixup(p, __fsm_sz); return __data_off(p + __fsm_sz); } return CSTR_NEQ; @@ -1870,8 +1903,10 @@ __parse_transfer_encoding(TfwHttpMsg *hm, unsigned char *data, size_t len, /* End of term. */ __FSM_STATE(I_EoT) { - if (IS_WS(c) || c == ',') - __FSM_I_MOVE(I_EoT); + if (c == ',') + __FSM_I_MOVE_fixup(I_EoT, 1, 0); + if (IS_WS(c)) + __FSM_I_MOVE_fixup(I_EoT, 1, TFW_STR_OWS); if (IS_TOKEN(c)) __FSM_I_JMP(I_TransEncodTok); if (IS_CRLF(c)) @@ -2410,7 +2445,7 @@ __parse_etag(TfwHttpMsg *hm, unsigned char *data, size_t len) /* End of token */ __FSM_STATE(I_EoT) { if (IS_WS(c)) - __FSM_I_MOVE_fixup(I_EoT, 1, 0); + __FSM_I_MOVE_fixup(I_EoT, 1, TFW_STR_OWS); if (IS_CRLF(c)) return __data_off(p); if ((TFW_CONN_TYPE(hm->conn) & Conn_Clnt) && c == ',') @@ -2420,7 +2455,7 @@ __parse_etag(TfwHttpMsg *hm, unsigned char *data, size_t len) __FSM_STATE(I_EoL) { if (IS_WS(c)) - __FSM_I_MOVE_fixup(I_EoL, 1, 0); + __FSM_I_MOVE_fixup(I_EoL, 1, TFW_STR_OWS); if (IS_CRLF(c)) return __data_off(p); return CSTR_NEQ; @@ -2890,7 +2925,8 @@ __parse_pragma(TfwHttpMsg *hm, unsigned char *data, size_t len) __FSM_START(parser->_i_st); __FSM_STATE(I_Pragma) { - TRY_STR("no-cache", I_Pragma, I_Pragma_NoCache); + TRY_STR_fixup(&TFW_STR_STRING("no-cache"), I_Pragma, + I_Pragma_NoCache); TRY_STR_INIT(); __FSM_I_JMP(I_Pragma_Ext); } @@ -2898,24 +2934,32 @@ __parse_pragma(TfwHttpMsg *hm, unsigned char *data, size_t len) __FSM_STATE(I_Pragma_NoCache) { if (IS_WS(c) || c == ',' || IS_CRLF(c)) msg->cache_ctl.flags |= TFW_HTTP_CC_PRAGMA_NO_CACHE; - __FSM_I_JMP(I_Pragma_Ext); + /* Fall through. */ } __FSM_STATE(I_Pragma_Ext) { /* Verify and just skip the extensions. */ - __FSM_I_MATCH_MOVE(qetoken, I_Pragma_Ext); + __FSM_I_MATCH_MOVE_fixup(qetoken, I_Pragma_Ext, 0); c = *(p + __fsm_sz); - if (IS_WS(c) || c == ',') - __FSM_I_MOVE_n(I_EoT, __fsm_sz + 1); - if (IS_CRLF(c)) + if (IS_WS(c) || c == ',') { + __msg_hdr_chunk_fixup(p, __fsm_sz); + p += __fsm_sz; + __FSM_I_JMP(I_EoT); + } + if (IS_CRLF(c)) { + __msg_hdr_chunk_fixup(p, __fsm_sz); return __data_off(p + __fsm_sz); + } + return CSTR_NEQ; } /* End of term. */ __FSM_STATE(I_EoT) { - if (IS_WS(c) || c == ',') - __FSM_I_MOVE(I_EoT); + if (IS_WS(c)) + __FSM_I_MOVE_fixup(I_EoT, 1, TFW_STR_OWS); + if (c == ',') + __FSM_I_MOVE_fixup(I_EoT, 1, 0); if (IS_CRLF(c)) return __data_off(p); __FSM_I_JMP(I_Pragma_Ext); @@ -3026,7 +3070,8 @@ __parse_keep_alive(TfwHttpMsg *hm, unsigned char *data, size_t len) __FSM_START(parser->_i_st); __FSM_STATE(I_KeepAlive) { - TRY_STR("timeout=", I_KeepAlive, I_KeepAliveTO); + TRY_STR_fixup(&TFW_STR_STRING("timeout="), I_KeepAlive, + I_KeepAliveTO); TRY_STR_INIT(); __FSM_I_JMP(I_KeepAliveExt); } @@ -3035,12 +3080,14 @@ __parse_keep_alive(TfwHttpMsg *hm, unsigned char *data, size_t len) __fsm_sz = __data_remain(p); __fsm_n = parse_int_list(p, __fsm_sz, &parser->_acc); if (__fsm_n == CSTR_POSTPONE) - __msg_hdr_chunk_fixup(data, len); + __msg_hdr_chunk_fixup(p, __fsm_sz); if (__fsm_n < 0) return __fsm_n; hm->keep_alive = parser->_acc; parser->_acc = 0; - __FSM_I_MOVE_n(I_EoT, __fsm_n); + __msg_hdr_chunk_fixup(p, __fsm_n); + p += __fsm_n; + __FSM_I_JMP(I_EoT); } /* @@ -3048,21 +3095,28 @@ __parse_keep_alive(TfwHttpMsg *hm, unsigned char *data, size_t len) * max=N */ __FSM_STATE(I_KeepAliveExt) { - __FSM_I_MATCH_MOVE(qetoken, I_KeepAliveExt); + __FSM_I_MATCH_MOVE_fixup(qetoken, I_KeepAliveExt, 0); c = *(p + __fsm_sz); - if (IS_WS(c) || c == ',') - __FSM_I_MOVE_n(I_EoT, __fsm_sz + 1); - if (IS_CRLF(c)) + if (IS_WS(c) || c == ',') { + __msg_hdr_chunk_fixup(p, __fsm_sz); + p += __fsm_sz; + __FSM_I_JMP(I_EoT); + } + if (IS_CRLF(c)) { + __msg_hdr_chunk_fixup(p, __fsm_sz); return __data_off(p + __fsm_sz); + } return CSTR_NEQ; } /* End of term. */ __FSM_STATE(I_EoT) { - if (IS_WS(c) || c == ',') - __FSM_I_MOVE(I_EoT); + if (c == ',') + __FSM_I_MOVE_fixup(I_EoT, 1, 0); + if (IS_WS(c)) + __FSM_I_MOVE_fixup(I_EoT, 1, TFW_STR_OWS); if (c == '=') - __FSM_I_MOVE(I_KeepAliveExt); + __FSM_I_MOVE_fixup(I_KeepAliveExt, 1, 0); if (IS_TOKEN(c)) __FSM_I_JMP(I_KeepAlive); if (IS_CRLF(c)) @@ -3551,7 +3605,7 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, __msg_field_finish_pos(&req->uri_path, p, 0); __FSM_MOVE_nofixup(Req_HttpVer); } - __FSM_MATCH_MOVE_pos_f(uri, Req_UriAbsPath, &req->uri_path); + __FSM_MATCH_MOVE_pos_f(uri, Req_UriAbsPath, &req->uri_path, 0); if (unlikely(*(p + __fsm_sz) != ' ')) TFW_PARSER_BLOCK(Req_UriAbsPath); __msg_field_finish_pos(&req->uri_path, p, __fsm_sz); @@ -3598,8 +3652,10 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && TFW_LC(*(p + 6)) == 't' && *(p + 13) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 6)); parser->_i_st = &&Req_HdrAcceptV; - __FSM_MOVE_n(RGen_LWS, 7); + p += 6; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } if (likely(__data_available(p, 14) && C8_INT_LCM(p + 1, 'u', 't', 'h', 'o', @@ -3607,8 +3663,10 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && C4_INT_LCM(p + 9, 't', 'i', 'o', 'n') && *(p + 13) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 13)); parser->_i_st = &&Req_HdrAuthorizationV; - __FSM_MOVE_n(RGen_LWS, 14); + p += 13; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Req_HdrA); case 'c': @@ -3623,8 +3681,11 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, 't', 'r', 'o', 'l', ':'))) { + __msg_hdr_chunk_fixup(data, + __data_off(p + 13)); parser->_i_st = &&Req_HdrCache_ControlV; - __FSM_MOVE_n(RGen_LWS, 14); + p += 13; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE_n(RGen_HdrOtherN, 5); case TFW_CHAR4_INT('o', 'n', 'n', 'e'): @@ -3632,8 +3693,11 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && TFW_LC(*(p + 9)) == 'n' && *(p + 10) == ':')) { + __msg_hdr_chunk_fixup(data, + __data_off(p + 10)); parser->_i_st = &&Req_HdrConnectionV; - __FSM_MOVE_n(RGen_LWS, 11); + p += 10; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE_n(RGen_HdrOtherN, 5); case TFW_CHAR4_INT('o', 'n', 't', 'e'): @@ -3648,8 +3712,11 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, if (likely(TFW_LC(*(p + 5)) == 'e' && *(p + 6) == ':')) { + __msg_hdr_chunk_fixup(data, + __data_off(p + 6)); parser->_i_st = &&Req_HdrCookieV; - __FSM_MOVE_n(RGen_LWS, 7); + p += 6; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE_n(RGen_HdrOtherN, 5); default: @@ -3659,9 +3726,11 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, if (likely(__data_available(p, 5) && C4_INT3_LCM(p + 1, 'o', 's', 't', ':'))) { + __msg_hdr_chunk_fixup(data, __data_off(p + 4)); parser->_i_st = &&Req_HdrHostV; parser->_hdr_tag = TFW_HTTP_HDR_HOST; - __FSM_MOVE_n(RGen_LWS, 5); + p += 4; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Req_HdrH); case 'i': @@ -3675,8 +3744,10 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && TFW_LC(*(p + 16)) == 'e' && *(p + 17) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 17)); parser->_i_st = &&Req_HdrIf_Modified_SinceV; - __FSM_MOVE_n(RGen_LWS, 18); + p += 17; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } if (likely(__data_available(p, 14) && TFW_LC(*(p + 1)) == 'f' @@ -3687,8 +3758,10 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && TFW_LC(*(p + 12)) == 'h' && *(p + 13) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 13)); parser->_i_st = &&Req_HdrIf_None_MatchV; - __FSM_MOVE_n(RGen_LWS, 14); + p += 13; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Req_HdrI); case 'k': @@ -3699,8 +3772,10 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && TFW_LC(*(p + 9)) == 'e' && *(p + 10) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 10)); parser->_i_st = &&Req_HdrKeep_AliveV; - __FSM_MOVE_n(RGen_LWS, 11); + p += 10; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Req_HdrK); case 'p': @@ -3709,8 +3784,10 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && TFW_LC(*(p + 5)) == 'a' && *(p + 6) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 6)); parser->_i_st = &&Req_HdrPragmaV; - __FSM_MOVE_n(RGen_LWS, 7); + p += 6; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Req_HdrP); case 'r': @@ -3720,8 +3797,10 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && TFW_LC(*(p + 6)) == 'r' && *(p + 7) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 7)); parser->_i_st = &&Req_HdrRefererV; - __FSM_MOVE_n(RGen_LWS, 8); + p += 7; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Req_HdrR); case 't': @@ -3733,8 +3812,10 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, 'd', 'i', 'n', 'g') && *(p + 17) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 17)); parser->_i_st = &&Req_HdrTransfer_EncodingV; - __FSM_MOVE_n(RGen_LWS, 18); + p += 17; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Req_HdrT); case 'x': @@ -3747,8 +3828,10 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && C8_INT7_LCM(p + 8, 'd', 'e', 'd', '-', 'f', 'o', 'r', ':'))) { + __msg_hdr_chunk_fixup(data, __data_off(p + 15)); parser->_i_st = &&Req_HdrX_Forwarded_ForV; - __FSM_MOVE_n(RGen_LWS, 16); + p += 15; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } if (likely(__data_available(p, 14) && *(p + 1) == '-' @@ -3760,8 +3843,10 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && TFW_LC(*(p + 12) == 'd') && *(p + 10) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 13)); parser->_i_st = &&Req_HdrX_Method_OverrideV; - __FSM_MOVE_n(RGen_LWS, 14); + p += 13; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } if (likely(__data_available(p, 23) && *(p + 1) == '-' @@ -3775,8 +3860,10 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && C8_INT7_LCM(p + 16, 'v', 'e', 'r', 'r', 'i', 'd', 'e', ':'))) { + __msg_hdr_chunk_fixup(data, __data_off(p + 22)); parser->_i_st = &&Req_HdrX_Method_OverrideV; - __FSM_MOVE_n(RGen_LWS, 23); + p += 22; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } if (likely(__data_available(p, 18) && *(p + 1) == '-' @@ -3787,8 +3874,10 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && C8_INT7_LCM(p + 10, 'v', 'e', 'r', 'r', 'i', 'd', 'e', ':'))) { + __msg_hdr_chunk_fixup(data, __data_off(p + 17)); parser->_i_st = &&Req_HdrX_Method_OverrideV; - __FSM_MOVE_n(RGen_LWS, 18); + p += 17; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Req_HdrX); case 'u': @@ -3799,8 +3888,10 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && TFW_LC(*(p + 9)) == 't' && *(p + 10) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 10)); parser->_i_st = &&Req_HdrUser_AgentV; - __FSM_MOVE_n(RGen_LWS, 11); + p += 10; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Req_HdrU); default: @@ -3817,16 +3908,20 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, && TFW_LC(*(p + 5)) == 'h' && *(p + 6) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 6)); parser->_i_st = &&Req_HdrContent_LengthV; - __FSM_MOVE_n(RGen_LWS, 7); + p += 6; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Req_HdrContent_L); case 't': if (likely(__data_available(p, 5) && C4_INT3_LCM(p + 1, 'y', 'p', 'e', ':'))) { + __msg_hdr_chunk_fixup(data, __data_off(p + 4)); parser->_i_st = &&Req_HdrContent_TypeV; - __FSM_MOVE_n(RGen_LWS, 5); + p += 4; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Req_HdrContent_T); default: @@ -3872,20 +3967,20 @@ tfw_http_parse_req(void *req_data, unsigned char *data, size_t len, __req_parse_if_msince); /* 'Keep-Alive:*OWS' is read, process field-value. */ - TFW_HTTP_PARSE_SPECHDR_VAL(Req_HdrKeep_AliveV, msg, __parse_keep_alive, - TFW_HTTP_HDR_KEEP_ALIVE); + __TFW_HTTP_PARSE_SPECHDR_VAL(Req_HdrKeep_AliveV, msg, __parse_keep_alive, + TFW_HTTP_HDR_KEEP_ALIVE, 0); /* 'Pragma:*OWS' is read, process field-value. */ - TFW_HTTP_PARSE_RAWHDR_VAL(Req_HdrPragmaV, msg, __parse_pragma); + __TFW_HTTP_PARSE_RAWHDR_VAL(Req_HdrPragmaV, msg, __parse_pragma, 0); /* 'Referer:*OWS' is read, process field-value. */ TFW_HTTP_PARSE_SPECHDR_VAL(Req_HdrRefererV, msg, __req_parse_referer, TFW_HTTP_HDR_REFERER); /* 'Transfer-Encoding:*OWS' is read, process field-value. */ - TFW_HTTP_PARSE_SPECHDR_VAL(Req_HdrTransfer_EncodingV, msg, - __req_parse_transfer_encoding, - TFW_HTTP_HDR_TRANSFER_ENCODING); + __TFW_HTTP_PARSE_SPECHDR_VAL(Req_HdrTransfer_EncodingV, msg, + __req_parse_transfer_encoding, + TFW_HTTP_HDR_TRANSFER_ENCODING, 0); /* 'X-Forwarded-For:*OWS' is read, process field-value. */ __TFW_HTTP_PARSE_SPECHDR_VAL(Req_HdrX_Forwarded_ForV, msg, @@ -8062,17 +8157,19 @@ __resp_parse_age(TfwHttpResp *resp, unsigned char *data, size_t len) __fsm_sz = __data_remain(p); __fsm_n = parse_int_ws(p, __fsm_sz, &parser->_acc); if (__fsm_n == CSTR_POSTPONE) - __msg_hdr_chunk_fixup(data, len); + __msg_hdr_chunk_fixup(p, __fsm_sz); if (__fsm_n < 0) return __fsm_n; resp->cache_ctl.age = parser->_acc; parser->_acc = 0; - __FSM_I_MOVE_n(Resp_I_EoL, __fsm_n); + __msg_hdr_chunk_fixup(p, __fsm_n); + p += __fsm_n; + /* Fall through. */ } __FSM_STATE(Resp_I_EoL) { if (IS_WS(c)) - __FSM_I_MOVE(Resp_I_EoL); + __FSM_I_MOVE_fixup(Resp_I_EoL, 1, TFW_STR_OWS); if (IS_CRLF(c)) { resp->cache_ctl.flags |= TFW_HTTP_CC_HDR_AGE; return __data_off(p); @@ -8112,8 +8209,10 @@ __resp_parse_cache_control(TfwHttpResp *resp, unsigned char *data, size_t len) } __FSM_STATE(Resp_I_CC_m) { - TRY_STR("max-age=", Resp_I_CC_m, Resp_I_CC_MaxAgeV); - TRY_STR_LAMBDA("must-revalidate", { + TRY_STR_fixup(&TFW_STR_STRING("max-age="), Resp_I_CC_m, + Resp_I_CC_MaxAgeV); + TRY_STR_LAMBDA_fixup(&TFW_STR_STRING("must-revalidate"), + &parser->hdr, { parser->_acc = TFW_HTTP_CC_MUST_REVAL; }, Resp_I_CC_m, Resp_I_Flag); TRY_STR_INIT(); @@ -8121,13 +8220,16 @@ __resp_parse_cache_control(TfwHttpResp *resp, unsigned char *data, size_t len) } __FSM_STATE(Resp_I_CC_n) { - TRY_STR_LAMBDA("no-cache", { + TRY_STR_LAMBDA_fixup(&TFW_STR_STRING("no-cache"), + &parser->hdr, { parser->_acc = TFW_HTTP_CC_NO_CACHE; }, Resp_I_CC_n, Resp_I_Flag); - TRY_STR_LAMBDA("no-store", { + TRY_STR_LAMBDA_fixup(&TFW_STR_STRING("no-store"), + &parser->hdr, { parser->_acc = TFW_HTTP_CC_NO_STORE; }, Resp_I_CC_n, Resp_I_Flag); - TRY_STR_LAMBDA("no-transform", { + TRY_STR_LAMBDA_fixup(&TFW_STR_STRING("no-transform"), + &parser->hdr, { parser->_acc = TFW_HTTP_CC_NO_TRANSFORM; }, Resp_I_CC_n, Resp_I_Flag); TRY_STR_INIT(); @@ -8135,13 +8237,14 @@ __resp_parse_cache_control(TfwHttpResp *resp, unsigned char *data, size_t len) } __FSM_STATE(Resp_I_CC_p) { - TRY_STR_LAMBDA("public", { + TRY_STR_LAMBDA_fixup(&TFW_STR_STRING("public"), &parser->hdr, { parser->_acc = TFW_HTTP_CC_PUBLIC; }, Resp_I_CC_p, Resp_I_Flag); - TRY_STR_LAMBDA("private", { + TRY_STR_LAMBDA_fixup(&TFW_STR_STRING("private"), &parser->hdr, { parser->_acc = TFW_HTTP_CC_PRIVATE; }, Resp_I_CC_p, Resp_I_Flag); - TRY_STR_LAMBDA("proxy-revalidate", { + TRY_STR_LAMBDA_fixup(&TFW_STR_STRING("proxy-revalidate"), + &parser->hdr, { parser->_acc = TFW_HTTP_CC_PROXY_REVAL; }, Resp_I_CC_p, Resp_I_Flag); TRY_STR_INIT(); @@ -8159,7 +8262,8 @@ __resp_parse_cache_control(TfwHttpResp *resp, unsigned char *data, size_t len) } __FSM_STATE(Resp_I_CC_s) { - TRY_STR("s-maxage=", Resp_I_CC_s, Resp_I_CC_SMaxAgeV); + TRY_STR_fixup(&TFW_STR_STRING("s-maxage="), Resp_I_CC_s, + Resp_I_CC_SMaxAgeV); TRY_STR_INIT(); __FSM_I_JMP(Resp_I_Ext); } @@ -8172,7 +8276,7 @@ __resp_parse_cache_control(TfwHttpResp *resp, unsigned char *data, size_t len) __fsm_sz = __data_remain(p); __fsm_n = parse_int_list(p, __fsm_sz, &parser->_acc); if (__fsm_n == CSTR_POSTPONE) - __msg_hdr_chunk_fixup(data, len); + __msg_hdr_chunk_fixup(p, __fsm_sz); if (__fsm_n < 0) { if (__fsm_n != CSTR_BADLEN) return __fsm_n; @@ -8180,7 +8284,7 @@ __resp_parse_cache_control(TfwHttpResp *resp, unsigned char *data, size_t len) } resp->cache_ctl.max_age = parser->_acc; resp->cache_ctl.flags |= TFW_HTTP_CC_MAX_AGE; - __FSM_I_MOVE_n(Resp_I_EoT, __fsm_n); + __FSM_I_MOVE_fixup(Resp_I_EoT, __fsm_n, 0); } __FSM_STATE(Resp_I_CC_SMaxAgeV) { @@ -8191,7 +8295,7 @@ __resp_parse_cache_control(TfwHttpResp *resp, unsigned char *data, size_t len) __fsm_sz = __data_remain(p); __fsm_n = parse_int_list(p, __fsm_sz, &parser->_acc); if (__fsm_n == CSTR_POSTPONE) - __msg_hdr_chunk_fixup(data, len); + __msg_hdr_chunk_fixup(p, __fsm_sz); if (__fsm_n < 0) { if (__fsm_n != CSTR_BADLEN) return __fsm_n; @@ -8199,17 +8303,21 @@ __resp_parse_cache_control(TfwHttpResp *resp, unsigned char *data, size_t len) } resp->cache_ctl.s_maxage = parser->_acc; resp->cache_ctl.flags |= TFW_HTTP_CC_S_MAXAGE; - __FSM_I_MOVE_n(Resp_I_EoT, __fsm_n); + __FSM_I_MOVE_fixup(Resp_I_EoT, __fsm_n, 0); } __FSM_STATE(Resp_I_Ext) { /* TODO: process cache extensions. */ - __FSM_I_MATCH_MOVE(qetoken, Resp_I_Ext); + __FSM_I_MATCH_MOVE_fixup(qetoken, Resp_I_Ext, 0); c = *(p + __fsm_sz); - if (IS_WS(c) || c == ',') - __FSM_I_MOVE_n(Resp_I_EoT, __fsm_sz + 1); + if (IS_WS(c) || c == ',') { + __msg_hdr_chunk_fixup(p, __fsm_sz); + p += __fsm_sz; + __FSM_I_JMP(Resp_I_EoT); + } if (IS_CRLF(c)) { parser->_acc = 0; + __msg_hdr_chunk_fixup(p, __fsm_sz); return __data_off(p + __fsm_sz); } return CSTR_NEQ; @@ -8217,8 +8325,11 @@ __resp_parse_cache_control(TfwHttpResp *resp, unsigned char *data, size_t len) /* End of term. */ __FSM_STATE(Resp_I_EoT) { - if (IS_WS(c) || c == ',') - __FSM_I_MOVE(Resp_I_EoT); + if (c == ',') + __FSM_I_MOVE_fixup(Resp_I_EoT, 1, 0); + + if (IS_WS(c)) + __FSM_I_MOVE_fixup(Resp_I_EoT, 1, TFW_STR_OWS); parser->_acc = 0; /* reinit for next token */ @@ -8228,7 +8339,7 @@ __resp_parse_cache_control(TfwHttpResp *resp, unsigned char *data, size_t len) * no-cache and private fields, so just skip '=[token]*'. */ if (c == '=') - __FSM_I_MOVE(Resp_I_Ext); + __FSM_I_MOVE_fixup(Resp_I_Ext, 1, 0); if (IS_TOKEN(c)) __FSM_I_JMP(Resp_I_CC); if (IS_CRLF(c)) @@ -8620,11 +8731,15 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, * reason-phrase = *( HTAB / SP / VCHAR / obs-text ) */ __FSM_STATE(Resp_ReasonPhrase) { - __FSM_MATCH_MOVE(ctext_vchar, Resp_ReasonPhrase); - p += __fsm_sz; - if (IS_CRLF(*p)) { + /* Store reason-phrase in separate chunk(s). */ + __msg_hdr_chunk_fixup(data, p - data); + __FSM_MATCH_MOVE_pos_f(ctext_vchar, Resp_ReasonPhrase, + &parser->hdr, TFW_STR_VALUE); + if (IS_CRLF(*(p + __fsm_sz))) { parser->_hdr_tag = TFW_HTTP_STATUS_LINE; - __msg_hdr_chunk_fixup(data, __data_off(p)); + __msg_hdr_chunk_fixup(p, __fsm_sz); + __msg_chunk_flags(TFW_STR_VALUE); + p += __fsm_sz; __FSM_JMP(RGen_EoL); } TFW_PARSER_BLOCK(Resp_ReasonPhrase); @@ -8653,9 +8768,11 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, '-', 'o', 'r', 'i') && C4_INT3_LCM(p + 24, 'g', 'i', 'n', ':'))) { + __msg_hdr_chunk_fixup(data, __data_off(p + 27)); parser->_i_st = &&RGen_HdrOtherV; __msg_hdr_set_hpack_index(20); - __FSM_MOVE_n(RGen_LWS, 28); + p += 27; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } if (likely(__data_available(p, 14) && C8_INT_LCM(p + 1, 'c', 'c', 'e', 'p', @@ -8663,24 +8780,30 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, && C4_INT_LCM(p + 9, 'n', 'g', 'e', 's') && *(p + 13) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 13)); parser->_i_st = &&RGen_HdrOtherV; __msg_hdr_set_hpack_index(18); - __FSM_MOVE_n(RGen_LWS, 14); + p += 13; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } if (likely(__data_available(p, 6) && C4_INT_LCM(p + 1, 'l', 'l', 'o', 'w') && *(p + 5) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 5)); parser->_i_st = &&RGen_HdrOtherV; __msg_hdr_set_hpack_index(22); - __FSM_MOVE_n(RGen_LWS, 6); + p += 5; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } if (likely(__data_available(p, 4) && C4_INT3_LCM(p, 'a', 'g', 'e', ':'))) { + __msg_hdr_chunk_fixup(data, __data_off(p + 3)); parser->_i_st = &&Resp_HdrAgeV; __msg_hdr_set_hpack_index(21); - __FSM_MOVE_n(RGen_LWS, 4); + p += 3; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Resp_HdrA); case 'c': @@ -8695,9 +8818,12 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, 't', 'r', 'o', 'l', ':'))) { + __msg_hdr_chunk_fixup(data, + __data_off(p + 13)); parser->_i_st = &&Resp_HdrCache_CtrlV; __msg_hdr_set_hpack_index(24); - __FSM_MOVE_n(RGen_LWS, 14); + p += 13; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE_n(RGen_HdrOtherN, 5); case TFW_CHAR4_INT('o', 'n', 'n', 'e'): @@ -8705,8 +8831,11 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, && TFW_LC(*(p + 9)) == 'n' && *(p + 10) == ':')) { + __msg_hdr_chunk_fixup(data, + __data_off(p + 10)); parser->_i_st = &&Resp_HdrConnectionV; - __FSM_MOVE_n(RGen_LWS, 11); + p += 10; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE_n(RGen_HdrOtherN, 5); case TFW_CHAR4_INT('o', 'n', 't', 'e'): @@ -8724,26 +8853,32 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, if (likely(__data_available(p, 5) && C4_INT3_LCM(p + 1, 'a', 't', 'e', ':'))) { + __msg_hdr_chunk_fixup(data, __data_off(p + 4)); parser->_i_st = &&Resp_HdrDateV; __msg_hdr_set_hpack_index(33); - __FSM_MOVE_n(RGen_LWS, 5); + p += 4; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Resp_HdrD); case 'e': if (likely(__data_available(p, 5) && C4_INT3_LCM(p + 1, 't', 'a', 'g', ':'))) { + __msg_hdr_chunk_fixup(data, __data_off(p + 4)); parser->_i_st = &&Resp_HdrEtagV; __msg_hdr_set_hpack_index(34); - __FSM_MOVE_n(RGen_LWS, 5); + p += 4; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } if (likely(__data_available(p, 8) && C8_INT7_LCM(p, 'e', 'x', 'p', 'i', 'r', 'e', 's', ':'))) { + __msg_hdr_chunk_fixup(data, __data_off(p + 7)); parser->_i_st = &&Resp_HdrExpiresV; __msg_hdr_set_hpack_index(36); - __FSM_MOVE_n(RGen_LWS, 8); + p += 7; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Resp_HdrE); case 'k': @@ -8754,8 +8889,10 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, && TFW_LC(*(p + 9)) == 'e' && *(p + 10) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 10)); parser->_i_st = &&Resp_HdrKeep_AliveV; - __FSM_MOVE_n(RGen_LWS, 11); + p += 10; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Resp_HdrK); case 'l': @@ -8766,24 +8903,30 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, 'f', 'i', 'e', 'd') && *(p + 13) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 13)); parser->_i_st = &&Resp_HdrLast_ModifiedV; __msg_hdr_set_hpack_index(44); - __FSM_MOVE_n(RGen_LWS, 14); + p += 13; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } if (likely(__data_available(p, 9) && C8_INT7_LCM(p + 1, 'o', 'c', 'a', 't', 'i', 'o', 'n', ':'))) { + __msg_hdr_chunk_fixup(data,__data_off(p + 8)); parser->_i_st = &&RGen_HdrOtherV; __msg_hdr_set_hpack_index(46); - __FSM_MOVE_n(RGen_LWS, 9); + p += 8; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } if (likely(__data_available(p, 5) && C4_INT3_LCM(p + 1, 'i', 'n', 'k', ':'))) { + __msg_hdr_chunk_fixup(data, __data_off(p + 4)); parser->_i_st = &&RGen_HdrOtherV; __msg_hdr_set_hpack_index(45); - __FSM_MOVE_n(RGen_LWS, 5); + p += 4; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Resp_HdrL); case 'p': @@ -8795,17 +8938,21 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, && TFW_LC(*(p + 17)) == 'e' && *(p + 18) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 18)); parser->_i_st = &&RGen_HdrOtherV; __msg_hdr_set_hpack_index(48); - __FSM_MOVE_n(RGen_LWS, 19); + p += 18; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } if (likely(__data_available(p, 7)) && C4_INT_LCM(p + 1, 'r', 'a', 'g', 'm') && TFW_LC(*(p + 5)) == 'a' && *(p + 6) == ':') { + __msg_hdr_chunk_fixup(data, __data_off(p + 6)); parser->_i_st = &&Resp_HdrPragmaV; - __FSM_MOVE_n(RGen_LWS, 7); + p += 6; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Resp_HdrP); case 'r': @@ -8814,9 +8961,11 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, 'y', '-', 'a', 'f') && C4_INT3_LCM(p + 8, 't', 'e', 'r', ':'))) { + __msg_hdr_chunk_fixup(data, __data_off(p + 11)); parser->_i_st = &&RGen_HdrOtherV; __msg_hdr_set_hpack_index(53); - __FSM_MOVE_n(RGen_LWS, 12); + p += 11; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Resp_HdrR); case 's': @@ -8829,9 +8978,11 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, 'r', 'i', 't', 'y') && *(p + 25) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 25)); parser->_i_st = &&RGen_HdrOtherV; __msg_hdr_set_hpack_index(56); - __FSM_MOVE_n(RGen_LWS, 26); + p += 25; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } if (likely(__data_available(p, 11) && C8_INT_LCM(p + 1, 'e', 't', '-', 'c', @@ -8840,18 +8991,22 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, && TFW_LC(*(p + 9)) == 'e' && *(p + 10) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 10)); parser->_i_st = &&Resp_HdrSet_CookieV; __msg_hdr_set_hpack_index(55); - __FSM_MOVE_n(RGen_LWS, 11); + p += 10; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } if (likely(__data_available(p, 7) && C4_INT_LCM(p + 1, 'e', 'r', 'v', 'e') && TFW_LC(*(p + 5)) == 'r' && *(p + 6) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 6)); parser->_i_st = &&Resp_HdrServerV; __msg_hdr_set_hpack_index(54); - __FSM_MOVE_n(RGen_LWS, 7); + p += 6; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Resp_HdrS); case 't': @@ -8863,24 +9018,30 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, 'd', 'i', 'n', 'g') && *(p + 17) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 17)); parser->_i_st = &&Resp_HdrTransfer_EncodingV; - __FSM_MOVE_n(RGen_LWS, 18); + p += 17; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Resp_HdrT); case 'v': if (likely(__data_available(p, 5) && C4_INT3_LCM(p + 1, 'a', 'r', 'y', ':'))) { + __msg_hdr_chunk_fixup(data, __data_off(p + 4)); parser->_i_st = &&RGen_HdrOtherV; __msg_hdr_set_hpack_index(59); - __FSM_MOVE_n(RGen_LWS, 5); + p += 4; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } if (likely(__data_available(p, 4) && C4_INT3_LCM(p, 'v', 'i', 'a', ':'))) { + __msg_hdr_chunk_fixup(data, __data_off(p + 3)); parser->_i_st = &&RGen_HdrOtherV; __msg_hdr_set_hpack_index(60); - __FSM_MOVE_n(RGen_LWS, 4); + p += 3; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Resp_HdrV); case 'w': @@ -8890,9 +9051,11 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, && C8_INT7_LCM(p + 9, 'n', 't', 'i', 'c', 'a', 't', 'e', ':'))) { + __msg_hdr_chunk_fixup(data, __data_off(p + 16)); parser->_i_st = &&RGen_HdrOtherV; __msg_hdr_set_hpack_index(61); - __FSM_MOVE_n(RGen_LWS, 17); + p += 16; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Resp_HdrW); default: @@ -8909,9 +9072,11 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, 'o', 's', 'i', 't') && C4_INT3_LCM(p + 8, 'i', 'o', 'n', ':'))) { + __msg_hdr_chunk_fixup(data, __data_off(p + 11)); parser->_i_st = &&RGen_HdrOtherV; __msg_hdr_set_hpack_index(25); - __FSM_MOVE_n(RGen_LWS, 12); + p += 11; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Resp_HdrContent_D); case 'e': @@ -8919,9 +9084,11 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, && C8_INT7_LCM(p + 1, 'n', 'c', 'o', 'd', 'i', 'n', 'g', ':'))) { + __msg_hdr_chunk_fixup(data, __data_off(p + 8)); parser->_i_st = &&RGen_HdrOtherV; __msg_hdr_set_hpack_index(26); - __FSM_MOVE_n(RGen_LWS, 9); + p += 8; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Resp_HdrContent_E); case 'l': @@ -8929,16 +9096,22 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, if (C8_INT7_LCM(p + 1, 'a', 'n', 'g', 'u', 'a', 'g', 'e', ':')) { + __msg_hdr_chunk_fixup(data, + __data_off(p + 8)); parser->_i_st = &&RGen_HdrOtherV; __msg_hdr_set_hpack_index(27); - __FSM_MOVE_n(RGen_LWS, 9); + p += 8; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } if (C8_INT7_LCM(p + 1, 'o', 'c', 'a', 't', 'i', 'o', 'n', ':')) { + __msg_hdr_chunk_fixup(data, + __data_off(p + 8)); parser->_i_st = &&RGen_HdrOtherV; __msg_hdr_set_hpack_index(29); - __FSM_MOVE_n(RGen_LWS, 9); + p += 8; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } } if (likely(__data_available(p, 7) @@ -8946,9 +9119,11 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, && TFW_LC(*(p + 5)) == 'h' && *(p + 6) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 6)); parser->_i_st = &&Resp_HdrContent_LengthV; __msg_hdr_set_hpack_index(28); - __FSM_MOVE_n(RGen_LWS, 7); + p += 6; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Resp_HdrContent_L); case 'r': @@ -8956,18 +9131,22 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, && C4_INT_LCM(p + 1, 'a', 'n', 'g', 'e') && *(p + 5) == ':')) { + __msg_hdr_chunk_fixup(data, __data_off(p + 5)); parser->_i_st = &&RGen_HdrOtherV; __msg_hdr_set_hpack_index(30); - __FSM_MOVE_n(RGen_LWS, 6); + p += 5; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Resp_HdrContent_R); case 't': if (likely(__data_available(p, 5) && C4_INT3_LCM(p + 1, 'y', 'p', 'e', ':'))) { + __msg_hdr_chunk_fixup(data, __data_off(p + 4)); parser->_i_st = &&Resp_HdrContent_TypeV; __msg_hdr_set_hpack_index(31); - __FSM_MOVE_n(RGen_LWS, 5); + p += 4; + __FSM_MOVE_hdr_fixup(RGen_LWS, 1); } __FSM_MOVE(Resp_HdrContent_T); default: @@ -8976,11 +9155,11 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, } /* 'Age:*OWS' is read, process field-value. */ - TFW_HTTP_PARSE_RAWHDR_VAL(Resp_HdrAgeV, resp, __resp_parse_age); + __TFW_HTTP_PARSE_RAWHDR_VAL(Resp_HdrAgeV, resp, __resp_parse_age, 0); /* 'Cache-Control:*OWS' is read, process field-value. */ - TFW_HTTP_PARSE_RAWHDR_VAL(Resp_HdrCache_CtrlV, resp, - __resp_parse_cache_control); + __TFW_HTTP_PARSE_RAWHDR_VAL(Resp_HdrCache_CtrlV, resp, + __resp_parse_cache_control, 0); /* 'Connection:*OWS' is read, process field-value. */ TFW_HTTP_PARSE_SPECHDR_VAL(Resp_HdrConnectionV, msg, __parse_connection, @@ -9007,24 +9186,24 @@ tfw_http_parse_resp(void *resp_data, unsigned char *data, size_t len, TFW_HTTP_PARSE_RAWHDR_VAL(Resp_HdrExpiresV, msg, __resp_parse_expires); /* 'Keep-Alive:*OWS' is read, process field-value. */ - TFW_HTTP_PARSE_SPECHDR_VAL(Resp_HdrKeep_AliveV, msg, __parse_keep_alive, - TFW_HTTP_HDR_KEEP_ALIVE); + __TFW_HTTP_PARSE_SPECHDR_VAL(Resp_HdrKeep_AliveV, msg, __parse_keep_alive, + TFW_HTTP_HDR_KEEP_ALIVE, 0); /* 'Last-Modified:*OWS' is read, process field-value. */ TFW_HTTP_PARSE_RAWHDR_VAL(Resp_HdrLast_ModifiedV, msg, __resp_parse_if_modified); /* 'Pragma:*OWS' is read, process field-value. */ - TFW_HTTP_PARSE_RAWHDR_VAL(Resp_HdrPragmaV, msg, __parse_pragma); + __TFW_HTTP_PARSE_RAWHDR_VAL(Resp_HdrPragmaV, msg, __parse_pragma, 0); /* 'Server:*OWS' is read, process field-value. */ TFW_HTTP_PARSE_SPECHDR_VAL(Resp_HdrServerV, resp, __resp_parse_server, TFW_HTTP_HDR_SERVER); /* 'Transfer-Encoding:*OWS' is read, process field-value. */ - TFW_HTTP_PARSE_SPECHDR_VAL(Resp_HdrTransfer_EncodingV, msg, - __resp_parse_transfer_encoding, - TFW_HTTP_HDR_TRANSFER_ENCODING); + __TFW_HTTP_PARSE_SPECHDR_VAL(Resp_HdrTransfer_EncodingV, msg, + __resp_parse_transfer_encoding, + TFW_HTTP_HDR_TRANSFER_ENCODING, 0); /* 'Set-Cookie:*OWS' is read, process field-value. */ __TFW_HTTP_PARSE_SPECHDR_VAL(Resp_HdrSet_CookieV, resp, diff --git a/tempesta_fw/http_sess.c b/tempesta_fw/http_sess.c index b2b1828902..2702386a14 100644 --- a/tempesta_fw/http_sess.c +++ b/tempesta_fw/http_sess.c @@ -206,15 +206,6 @@ tfw_http_sticky_build_redirect(TfwHttpReq *req, StickyVal *sv, RedirMarkVal *mv) if (!tfw_http_sticky_redirect_applied(req)) return TFW_HTTP_SESS_JS_NOT_SUPPORTED; - if (TFW_MSG_H2(req)) { - /* - * TODO #309: add separate flow for HTTP/2 response preparing - * and sending (HPACK index, encode in HTTP/2 format, add frame - * headers and send via @tfw_h2_resp_fwd()). - */ - return TFW_HTTP_SESS_REDIRECT_NEED; - } - if (!(resp = tfw_http_msg_alloc_resp_light(req))) return TFW_HTTP_SESS_FAILURE; @@ -250,8 +241,11 @@ tfw_http_sticky_build_redirect(TfwHttpReq *req, StickyVal *sv, RedirMarkVal *mv) cookie.nchunks++; } - r = tfw_http_prep_redirect((TfwHttpMsg *)resp, sticky->redirect_code, - &rmark, &cookie, body); + r = TFW_MSG_H2(req) + ? tfw_h2_prep_redirect(resp, sticky->redirect_code, &rmark, + &cookie, body) + : tfw_h1_prep_redirect(resp, sticky->redirect_code, &rmark, + &cookie, body); if (r) { tfw_http_msg_free((TfwHttpMsg *)resp); return TFW_HTTP_SESS_FAILURE; @@ -424,11 +418,11 @@ tfw_http_sticky_calc(TfwHttpReq *req, StickyVal *sv) /* * Add Tempesta sticky cookie to an HTTP response. * - * Create a complete 'Set-Cookie:' header field, and add it + * Create a complete 'set-cookie' header field, and add it * to the HTTP response' header block. */ static int -tfw_http_sticky_add(TfwHttpResp *resp) +tfw_http_sticky_add(TfwHttpResp *resp, bool cache) { int r; static const unsigned int len = sizeof(StickyVal) * 2; @@ -460,8 +454,21 @@ tfw_http_sticky_add(TfwHttpResp *resp) PR_TFW_STR(&sticky->name), len, buf); if (to_h2) { + TfwH2TransOp op = cache ? TFW_H2_TRANS_EXPAND : TFW_H2_TRANS_ADD; + set_cookie.hpack_idx = 55; - r = __hdr_h2_add(resp, &set_cookie); + r = tfw_hpack_encode(resp, &set_cookie, op, !cache); + } + else if (cache) { + TfwHttpTransIter *mit = &resp->mit; + struct sk_buff **skb_head = &resp->msg.skb_head; + TfwStr crlf = { .data = S_CRLF, .len = SLEN(S_CRLF) }; + + r = tfw_http_msg_expand_data(&mit->iter, skb_head, + &set_cookie, NULL); + if (!r) + r = tfw_http_msg_expand_data(&mit->iter, skb_head, + &crlf, NULL); } else { r = tfw_http_msg_hdr_add((TfwHttpMsg *)resp, &set_cookie); @@ -825,7 +832,7 @@ tfw_http_sess_req_process(TfwHttpReq *req) * Add Tempesta sticky cookie to an HTTP response if needed. */ int -tfw_http_sess_resp_process(TfwHttpResp *resp) +tfw_http_sess_resp_process(TfwHttpResp *resp, bool cache) { TfwHttpReq *req = resp->req; TfwStickyCookie *sticky = req->vhost->cookie; @@ -846,7 +853,7 @@ tfw_http_sess_resp_process(TfwHttpResp *resp) */ if (test_bit(TFW_HTTP_B_HAS_STICKY, req->flags)) return 0; - return tfw_http_sticky_add(resp); + return tfw_http_sticky_add(resp, cache); } /** diff --git a/tempesta_fw/http_sess.h b/tempesta_fw/http_sess.h index 513bf8ae9e..fce10917aa 100644 --- a/tempesta_fw/http_sess.h +++ b/tempesta_fw/http_sess.h @@ -168,7 +168,7 @@ enum { int tfw_http_sess_obtain(TfwHttpReq *req); void tfw_http_sess_learn(TfwHttpResp *resp); int tfw_http_sess_req_process(TfwHttpReq *req); -int tfw_http_sess_resp_process(TfwHttpResp *resp); +int tfw_http_sess_resp_process(TfwHttpResp *resp, bool cache); void tfw_http_sess_put(TfwHttpSess *sess); void tfw_http_sess_pin_vhost(TfwHttpSess *sess, TfwVhost *vhost); diff --git a/tempesta_fw/http_stream.c b/tempesta_fw/http_stream.c index 6a0f8357c6..f98c186fbc 100644 --- a/tempesta_fw/http_stream.c +++ b/tempesta_fw/http_stream.c @@ -134,7 +134,7 @@ tfw_h2_stream_fsm(TfwStream *stream, unsigned char type, unsigned char flags, } /* * Received RST_STREAM frame immediately moves stream into the - * final 'closed' state, while the the sent RST_STREAM moves stream + * final 'closed' state, while the sent RST_STREAM moves stream * into the intermediate 'locally closed' state. */ else if (type == HTTP2_RST_STREAM) { diff --git a/tempesta_fw/http_types.h b/tempesta_fw/http_types.h index 18a8d60967..4ef7a380a7 100644 --- a/tempesta_fw/http_types.h +++ b/tempesta_fw/http_types.h @@ -26,6 +26,8 @@ typedef struct tfw_http_msg_t TfwHttpMsg; typedef struct tfw_http_req_t TfwHttpReq; typedef struct tfw_http_resp_t TfwHttpResp; typedef struct tfw_vhost_t TfwVhost; +typedef struct tfw_hdr_mods_desc_t TfwHdrModsDesc; +typedef struct tfw_hdr_mods_t TfwHdrMods; typedef struct frang_global_cfg_t FrangGlobCfg; typedef struct frang_vhost_cfg_t FrangVhostCfg; typedef struct tfw_http_cookie_t TfwStickyCookie; diff --git a/tempesta_fw/str.c b/tempesta_fw/str.c index 7c3f691121..4a4c53837b 100644 --- a/tempesta_fw/str.c +++ b/tempesta_fw/str.c @@ -771,6 +771,32 @@ tfw_strcpy_desc(TfwStr *dst, TfwStr *src) } EXPORT_SYMBOL(tfw_strcpy_desc); +/** + * Same as @tfw_strcpy_desc(), but allocate chunk descriptors for result string. + */ +TfwStr * +tfw_strdup_desc(TfwPool *pool, const TfwStr *src) +{ + size_t sz; + TfwStr *dst, *d; + const TfwStr *s, *end; + + sz = (src->nchunks + 1) * sizeof(TfwStr); + if (!(dst = (TfwStr *)tfw_pool_alloc(pool, sz))) + return NULL; + + *dst = *src; + dst->chunks = dst + 1; + + d = TFW_STR_CHUNK(dst, 0); + TFW_STR_FOR_EACH_CHUNK(s, src, end) { + *d = *s; + ++d; + } + + return dst; +} + static inline int __tfw_str_insert(TfwPool *pool, TfwStr *dst, TfwStr *src, unsigned int chunk) { diff --git a/tempesta_fw/str.h b/tempesta_fw/str.h index 405bc9c847..28b42d8526 100644 --- a/tempesta_fw/str.h +++ b/tempesta_fw/str.h @@ -215,6 +215,9 @@ size_t tfw_ultohex(unsigned long ai, char *buf, unsigned int len); */ #define TFW_STR_HDR_VALUE 0x80 +/* The chunk contains only WS characters. */ +#define TFW_STR_OWS 0x100 + /* * @ptr - pointer to string data or array of nested strings; * @skb - socket buffer containing the string data; @@ -390,6 +393,7 @@ typedef enum { int tfw_strcpy(TfwStr *dst, const TfwStr *src); TfwStr *tfw_strdup(TfwPool *pool, const TfwStr *src); int tfw_strcpy_desc(TfwStr *dst, TfwStr *src); +TfwStr *tfw_strdup_desc(TfwPool *pool, const TfwStr *src); int tfw_strcat(TfwPool *pool, TfwStr *dst, TfwStr *src); int tfw_str_insert(TfwPool *pool, TfwStr *dst, TfwStr *src, unsigned int chunk); diff --git a/tempesta_fw/t/unit/test_hpack.c b/tempesta_fw/t/unit/test_hpack.c index b0c72ea7ef..c410f6a140 100644 --- a/tempesta_fw/t/unit/test_hpack.c +++ b/tempesta_fw/t/unit/test_hpack.c @@ -92,6 +92,34 @@ test_h2_teardown(void) free_all_str(); } +static void +test_h2_hdr_name(TfwStr *hdr, TfwStr *out_name) +{ + const TfwStr *c, *end; + + if (unlikely(TFW_STR_EMPTY(hdr))) { + TFW_STR_INIT(out_name); + return; + } + + BUG_ON(TFW_STR_DUP(hdr)); + BUG_ON(TFW_STR_EMPTY(hdr)); + + *out_name = *hdr; + + if (unlikely(TFW_STR_PLAIN(hdr))) { + WARN_ON_ONCE(hdr->flags & TFW_STR_HDR_VALUE); + return; + } + + TFW_STR_FOR_EACH_CHUNK(c, hdr, end) { + if (c->flags & TFW_STR_HDR_VALUE) { + out_name->len -= c->len; + out_name->nchunks--; + } + } +} + TEST(hpack, dec_table_static) { TfwHPack *hp; @@ -186,21 +214,21 @@ TEST(hpack, dec_table_dynamic) hp = &ctx.hpack; - __h2_msg_hdr_name(s1, &h_name); + test_h2_hdr_name(s1, &h_name); it->nm_num = h_name.nchunks; it->nm_len = h_name.len; it->tag = TFW_TAG_HDR_RAW; *it->parsed_hdr = *s1; EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, it)); - __h2_msg_hdr_name(s2, &h_name); + test_h2_hdr_name(s2, &h_name); it->nm_num = h_name.nchunks; it->nm_len = h_name.len; it->tag = TFW_TAG_HDR_X_FORWARDED_FOR; *it->parsed_hdr = *s2; EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, it)); - __h2_msg_hdr_name(s3, &h_name); + test_h2_hdr_name(s3, &h_name); it->nm_num = h_name.nchunks; it->nm_len = h_name.len; it->tag = TFW_TAG_HDR_RAW; @@ -254,14 +282,14 @@ TEST(hpack, dec_table_dynamic_inc) hp = &ctx.hpack; - __h2_msg_hdr_name(s1, &h_name); + test_h2_hdr_name(s1, &h_name); it->nm_num = h_name.nchunks; it->nm_len = h_name.len; it->tag = TFW_TAG_HDR_RAW; *it->parsed_hdr = *s1; EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, it)); - __h2_msg_hdr_name(s2, &h_name); + test_h2_hdr_name(s2, &h_name); it->nm_num = h_name.nchunks; it->nm_len = h_name.len; it->tag = TFW_TAG_HDR_RAW; @@ -278,14 +306,14 @@ TEST(hpack, dec_table_dynamic_inc) if (entry) EXPECT_TRUE(tfw_strcmp(entry->hdr, s1) == 0); - __h2_msg_hdr_name(s3, &h_name); + test_h2_hdr_name(s3, &h_name); it->nm_num = h_name.nchunks; it->nm_len = h_name.len; it->tag = TFW_TAG_HDR_CACHE_CONTROL; *it->parsed_hdr = *s3; EXPECT_OK(tfw_hpack_add_index(&hp->dec_tbl, it)); - __h2_msg_hdr_name(s4, &h_name); + test_h2_hdr_name(s4, &h_name); it->nm_num = h_name.nchunks; it->nm_len = h_name.len; it->tag = TFW_TAG_HDR_RAW; @@ -302,7 +330,7 @@ TEST(hpack, dec_table_dynamic_inc) if (entry) EXPECT_TRUE(tfw_strcmp(entry->hdr, s3) == 0); - __h2_msg_hdr_name(s5, &h_name); + test_h2_hdr_name(s5, &h_name); it->nm_num = h_name.nchunks; it->nm_len = h_name.len; it->tag = TFW_TAG_HDR_RAW; @@ -415,7 +443,7 @@ TEST(hpack, dec_table_wrap) EXPECT_NOT_NULL(l_entry->hdr); EXPECT_NOT_NULL(t_entry->hdr); if (l_entry->hdr) { - __h2_msg_hdr_name(t_entry->hdr, &h_name); + test_h2_hdr_name(t_entry->hdr, &h_name); EXPECT_TRUE(tfw_strcmp(&h_name, l_entry->hdr) == 0); } } @@ -524,7 +552,7 @@ TEST(hpack, dec_raw) EXPECT_NE(ht, test_req->h_tbl); ht = test_req->h_tbl; - __h2_msg_hdr_name(&ht->tbl[TFW_HTTP_HDR_RAW], &h_name); + test_h2_hdr_name(&ht->tbl[TFW_HTTP_HDR_RAW], &h_name); __h2_msg_hdr_val(&ht->tbl[TFW_HTTP_HDR_RAW], &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); @@ -537,7 +565,7 @@ TEST(hpack, dec_raw) strlen(test_value1), 0)); } - __h2_msg_hdr_name(&ht->tbl[TFW_HTTP_HDR_RAW + 1], &h_name); + test_h2_hdr_name(&ht->tbl[TFW_HTTP_HDR_RAW + 1], &h_name); __h2_msg_hdr_val(&ht->tbl[TFW_HTTP_HDR_RAW + 1], &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); @@ -550,7 +578,7 @@ TEST(hpack, dec_raw) strlen(test_value2), 0)); } - __h2_msg_hdr_name(&ht->tbl[TFW_HTTP_HDR_X_FORWARDED_FOR], &h_name); + test_h2_hdr_name(&ht->tbl[TFW_HTTP_HDR_X_FORWARDED_FOR], &h_name); __h2_msg_hdr_val(&ht->tbl[TFW_HTTP_HDR_X_FORWARDED_FOR], &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); @@ -747,7 +775,7 @@ TEST(hpack, dec_indexed) EXPECT_EQ(hdr->nchunks, 3); if (hdr->nchunks == 3) { dup = hdr->chunks; - __h2_msg_hdr_name(dup, &h_name); + test_h2_hdr_name(dup, &h_name); __h2_msg_hdr_val(dup, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); @@ -760,7 +788,7 @@ TEST(hpack, dec_indexed) test_len_val1, 0)); } dup = hdr->chunks + 1; - __h2_msg_hdr_name(dup, &h_name); + test_h2_hdr_name(dup, &h_name); __h2_msg_hdr_val(dup, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); @@ -773,7 +801,7 @@ TEST(hpack, dec_indexed) test_len_val1, 0)); } dup = hdr->chunks + 2; - __h2_msg_hdr_name(dup, &h_name); + test_h2_hdr_name(dup, &h_name); __h2_msg_hdr_val(dup, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); @@ -792,7 +820,7 @@ TEST(hpack, dec_indexed) EXPECT_EQ(hdr->nchunks, 2); if (hdr->nchunks == 2) { dup = hdr->chunks; - __h2_msg_hdr_name(dup, &h_name); + test_h2_hdr_name(dup, &h_name); __h2_msg_hdr_val(dup, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); @@ -805,7 +833,7 @@ TEST(hpack, dec_indexed) test_len_val2, 0)); } dup = hdr->chunks + 1; - __h2_msg_hdr_name(dup, &h_name); + test_h2_hdr_name(dup, &h_name); __h2_msg_hdr_val(dup, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); @@ -820,7 +848,7 @@ TEST(hpack, dec_indexed) } hdr = &ht->tbl[TFW_HTTP_HDR_HOST]; - __h2_msg_hdr_name(hdr, &h_name); + test_h2_hdr_name(hdr, &h_name); __h2_msg_hdr_val(hdr, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); @@ -834,7 +862,7 @@ TEST(hpack, dec_indexed) } hdr = &ht->tbl[TFW_HTTP_HDR_REFERER]; - __h2_msg_hdr_name(hdr, &h_name); + test_h2_hdr_name(hdr, &h_name); __h2_msg_hdr_val(hdr, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); @@ -863,7 +891,7 @@ TEST(hpack, dec_indexed) EXPECT_NOT_NULL(entry); if (entry) { hdr = entry->hdr; - __h2_msg_hdr_name(hdr, &h_name); + test_h2_hdr_name(hdr, &h_name); __h2_msg_hdr_val(hdr, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); @@ -884,7 +912,7 @@ TEST(hpack, dec_indexed) EXPECT_NOT_NULL(entry); if (entry) { hdr = entry->hdr; - __h2_msg_hdr_name(hdr, &h_name); + test_h2_hdr_name(hdr, &h_name); __h2_msg_hdr_val(hdr, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); @@ -905,7 +933,7 @@ TEST(hpack, dec_indexed) EXPECT_NOT_NULL(entry); if (entry) { hdr = entry->hdr; - __h2_msg_hdr_name(hdr, &h_name); + test_h2_hdr_name(hdr, &h_name); __h2_msg_hdr_val(hdr, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); @@ -1047,7 +1075,7 @@ TEST(hpack, dec_huffman) * into the headers table. */ hdr = &ht->tbl[TFW_HTTP_HDR_H2_AUTHORITY]; - __h2_msg_hdr_name(hdr, &h_name); + test_h2_hdr_name(hdr, &h_name); __h2_msg_hdr_val(hdr, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); @@ -1061,7 +1089,7 @@ TEST(hpack, dec_huffman) } hdr = &ht->tbl[TFW_HTTP_HDR_RAW]; - __h2_msg_hdr_name(hdr, &h_name); + test_h2_hdr_name(hdr, &h_name); __h2_msg_hdr_val(hdr, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); @@ -1075,7 +1103,7 @@ TEST(hpack, dec_huffman) } hdr = &ht->tbl[TFW_HTTP_HDR_RAW + 1]; - __h2_msg_hdr_name(hdr, &h_name); + test_h2_hdr_name(hdr, &h_name); __h2_msg_hdr_val(hdr, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); @@ -1096,7 +1124,7 @@ TEST(hpack, dec_huffman) EXPECT_NOT_NULL(entry); if (entry) { hdr = entry->hdr; - __h2_msg_hdr_name(hdr, &h_name); + test_h2_hdr_name(hdr, &h_name); __h2_msg_hdr_val(hdr, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); @@ -1117,7 +1145,7 @@ TEST(hpack, dec_huffman) EXPECT_NOT_NULL(entry); if (entry) { hdr = entry->hdr; - __h2_msg_hdr_name(hdr, &h_name); + test_h2_hdr_name(hdr, &h_name); __h2_msg_hdr_val(hdr, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); @@ -1138,7 +1166,7 @@ TEST(hpack, dec_huffman) EXPECT_NOT_NULL(entry); if (entry) { hdr = entry->hdr; - __h2_msg_hdr_name(hdr, &h_name); + test_h2_hdr_name(hdr, &h_name); __h2_msg_hdr_val(hdr, &h_value); EXPECT_TRUE(!TFW_STR_EMPTY(&h_name)); EXPECT_TRUE(!TFW_STR_EMPTY(&h_value)); @@ -1205,8 +1233,9 @@ TEST(hpack, enc_huffman) TEST(hpack, enc_table_hdr_write) { - char *buf; - unsigned long hdr_len, n_len, v_off, v_len; + char *buf, *ptr; + unsigned long hdr_len; + TfwStr s_nm = {}, s_val = {}; #define HDR_NAME_1 "x-forwarded-for" #define HDR_VALUE_1 "test.com, foo.com, example.com" @@ -1219,103 +1248,116 @@ TEST(hpack, enc_table_hdr_write) #define HDR_NAME_5 "custom-key" #define HDR_VALUE_5 "custom-example-value" - TFW_STR(s1, HDR_NAME_1 ":"); - TFW_STR(s1_lws, " "); - TFW_STR(s1_value, HDR_VALUE_1 " "); + TFW_STR(col, ":"); + + TFW_STR(s1, HDR_NAME_1); + TFW_STR(s1_lws, " "); + TFW_STR(s1_value, HDR_VALUE_1); + TFW_STR(s1_rws, " "); const char *t_s1 = HDR_NAME_1 HDR_VALUE_1; - unsigned long off1 = s1_lws->len + 1; unsigned long t_s1_len = strlen(t_s1); - TFW_STR(s2, HDR_NAME_2 ":"); + TFW_STR(s2, HDR_NAME_2); TFW_STR(s2_value, HDR_VALUE_2); const char *t_s2 = HDR_NAME_2 HDR_VALUE_2; - unsigned long off2 = 1; unsigned long t_s2_len = strlen(t_s2); - TFW_STR(s3, HDR_NAME_3 ":"); + TFW_STR(s3, HDR_NAME_3); TFW_STR(s3_lws, "\t "); - TFW_STR(s3_value, HDR_VALUE_3 " "); + TFW_STR(s3_value, HDR_VALUE_3); + TFW_STR(s3_rws, " "); const char *t_s3 = HDR_NAME_3 HDR_VALUE_3; - unsigned long off3 = s3_lws->len + 1; unsigned long t_s3_len = strlen(t_s3); - TFW_STR(s4, HDR_NAME_4 ":"); + TFW_STR(s4, HDR_NAME_4); TFW_STR(s4_lws, " "); - TFW_STR(s4_value, HDR_VALUE_4 "\t\t \t"); + TFW_STR(s4_value, HDR_VALUE_4); + TFW_STR(s4_rws, "\t\t \t"); const char *t_s4 = HDR_NAME_4 HDR_VALUE_4; - unsigned long off4 = s4_lws->len + 1; unsigned long t_s4_len = strlen(t_s4); - TFW_STR(s5, HDR_NAME_5 ":"); + TFW_STR(s5, HDR_NAME_5); TFW_STR(s5_lws, "\t\t\t"); - TFW_STR(s5_value, HDR_VALUE_5 "\t\t\t\t"); + TFW_STR(s5_value, HDR_VALUE_5); + TFW_STR(s5_rws, "\t\t\t\t"); const char *t_s5 = HDR_NAME_5 HDR_VALUE_5; - unsigned long off5 = s5_lws->len + 1; unsigned long t_s5_len = strlen(t_s5); - collect_compound_str(s1, s1_lws, 0); + collect_compound_str(s1, col, 0); + collect_compound_str(s1, s1_lws, TFW_STR_OWS); collect_compound_str(s1, s1_value, 0); + collect_compound_str(s1, s1_rws, TFW_STR_OWS); + collect_compound_str(s2, col, 0); collect_compound_str(s2, s2_value, 0); - collect_compound_str(s3, s3_lws, 0); + collect_compound_str(s3, col, 0); + collect_compound_str(s3, s3_lws, TFW_STR_OWS); collect_compound_str(s3, s3_value, 0); - collect_compound_str(s4, s4_lws, 0); + collect_compound_str(s3, s3_rws, TFW_STR_OWS); + collect_compound_str(s4, col, 0); + collect_compound_str(s4, s4_lws, TFW_STR_OWS); collect_compound_str(s4, s4_value, 0); - collect_compound_str(s5, s5_lws, 0); + collect_compound_str(s4, s4_rws, TFW_STR_OWS); + collect_compound_str(s5, col, 0); + collect_compound_str(s5, s5_lws, TFW_STR_OWS); collect_compound_str(s5, s5_value, 0); + collect_compound_str(s5, s5_rws, TFW_STR_OWS); - hdr_len = tfw_h2_msg_hdr_length(s1, &n_len, &v_off, &v_len, - TFW_H2_TRANS_INPLACE); - EXPECT_EQ(n_len, strlen(HDR_NAME_1)); - EXPECT_EQ(v_len, strlen(HDR_VALUE_1)); - EXPECT_EQ(v_off, off1); + hdr_len = tfw_http_hdr_split(s1, &s_nm, &s_val, true); + EXPECT_EQ(s_nm.len, strlen(HDR_NAME_1)); + EXPECT_EQ(s_val.len, strlen(HDR_VALUE_1)); EXPECT_EQ(hdr_len, t_s1_len); buf = tfw_pool_alloc(str_pool, hdr_len); BUG_ON(!buf); - tfw_h2_msg_hdr_write(s1, n_len, v_off, v_len, buf); + ptr = tfw_hpack_write(&s_nm, buf); + tfw_hpack_write(&s_val, ptr); EXPECT_OK(memcmp_fast(t_s1, buf, hdr_len)); - hdr_len = tfw_h2_msg_hdr_length(s2, &n_len, &v_off, &v_len, - TFW_H2_TRANS_INPLACE); - EXPECT_EQ(n_len, strlen(HDR_NAME_2)); - EXPECT_EQ(v_len, strlen(HDR_VALUE_2)); - EXPECT_EQ(v_off, off2); + TFW_STR_INIT(&s_nm); + TFW_STR_INIT(&s_val); + hdr_len = tfw_http_hdr_split(s2, &s_nm, &s_val, true); + EXPECT_EQ(s_nm.len, strlen(HDR_NAME_2)); + EXPECT_EQ(s_val.len, strlen(HDR_VALUE_2)); EXPECT_EQ(hdr_len, t_s2_len); buf = tfw_pool_alloc(str_pool, hdr_len); BUG_ON(!buf); - tfw_h2_msg_hdr_write(s2, n_len, v_off, v_len, buf); + ptr = tfw_hpack_write(&s_nm, buf); + tfw_hpack_write(&s_val, ptr); EXPECT_OK(memcmp_fast(t_s2, buf, hdr_len)); - hdr_len = tfw_h2_msg_hdr_length(s3, &n_len, &v_off, &v_len, - TFW_H2_TRANS_INPLACE); - EXPECT_EQ(n_len, strlen(HDR_NAME_3)); - EXPECT_EQ(v_len, strlen(HDR_VALUE_3)); - EXPECT_EQ(v_off, off3); + TFW_STR_INIT(&s_nm); + TFW_STR_INIT(&s_val); + hdr_len = tfw_http_hdr_split(s3, &s_nm, &s_val, true); + EXPECT_EQ(s_nm.len, strlen(HDR_NAME_3)); + EXPECT_EQ(s_val.len, strlen(HDR_VALUE_3)); EXPECT_EQ(hdr_len, t_s3_len); buf = tfw_pool_alloc(str_pool, hdr_len); BUG_ON(!buf); - tfw_h2_msg_hdr_write(s3, n_len, v_off, v_len, buf); + ptr = tfw_hpack_write(&s_nm, buf); + tfw_hpack_write(&s_val, ptr); EXPECT_OK(memcmp_fast(t_s3, buf, hdr_len)); - hdr_len = tfw_h2_msg_hdr_length(s4, &n_len, &v_off, &v_len, - TFW_H2_TRANS_INPLACE); - EXPECT_EQ(n_len, strlen(HDR_NAME_4)); - EXPECT_EQ(v_len, strlen(HDR_VALUE_4)); - EXPECT_EQ(v_off, off4); + TFW_STR_INIT(&s_nm); + TFW_STR_INIT(&s_val); + hdr_len = tfw_http_hdr_split(s4, &s_nm, &s_val, true); + EXPECT_EQ(s_nm.len, strlen(HDR_NAME_4)); + EXPECT_EQ(s_val.len, strlen(HDR_VALUE_4)); EXPECT_EQ(hdr_len, t_s4_len); buf = tfw_pool_alloc(str_pool, hdr_len); BUG_ON(!buf); - tfw_h2_msg_hdr_write(s4, n_len, v_off, v_len, buf); + ptr = tfw_hpack_write(&s_nm, buf); + tfw_hpack_write(&s_val, ptr); EXPECT_OK(memcmp_fast(t_s4, buf, hdr_len)); - hdr_len = tfw_h2_msg_hdr_length(s5, &n_len, &v_off, &v_len, - TFW_H2_TRANS_INPLACE); - EXPECT_EQ(n_len, strlen(HDR_NAME_5)); - EXPECT_EQ(v_len, strlen(HDR_VALUE_5)); - EXPECT_EQ(v_off, off5); + TFW_STR_INIT(&s_nm); + TFW_STR_INIT(&s_val); + hdr_len = tfw_http_hdr_split(s5, &s_nm, &s_val, true); + EXPECT_EQ(s_nm.len, strlen(HDR_NAME_5)); + EXPECT_EQ(s_val.len, strlen(HDR_VALUE_5)); EXPECT_EQ(hdr_len, t_s5_len); buf = tfw_pool_alloc(str_pool, hdr_len); BUG_ON(!buf); - tfw_h2_msg_hdr_write(s5, n_len, v_off, v_len, buf); + ptr = tfw_hpack_write(&s_nm, buf); + tfw_hpack_write(&s_val, ptr); EXPECT_OK(memcmp_fast(t_s5, buf, hdr_len)); #undef HDR_NAME_1 @@ -1345,28 +1387,37 @@ TEST(hpack, enc_table_index) #define HDR_NAME_3 "test-example-key" #define HDR_VALUE_3 "custom-example-value" - TFW_STR(s1, HDR_NAME_1 ":"); + TFW_STR(col, ":"); + + TFW_STR(s1, HDR_NAME_1); TFW_STR(s1_lws, " \t "); - TFW_STR(s1_value, HDR_VALUE_1 " "); + TFW_STR(s1_value, HDR_VALUE_1); + TFW_STR(s1_rws, " "); const char *t_s1 = HDR_NAME_1 HDR_VALUE_1; unsigned long t_s1_len = strlen(t_s1); - TFW_STR(s2, HDR_NAME_2 ":"); + TFW_STR(s2, HDR_NAME_2); TFW_STR(s2_value, HDR_VALUE_2); const char *t_s2 = HDR_NAME_2 HDR_VALUE_2; unsigned long t_s2_len = strlen(t_s2); - TFW_STR(s3, HDR_NAME_3 ":"); + TFW_STR(s3, HDR_NAME_3); TFW_STR(s3_lws, "\t \t\t\t"); - TFW_STR(s3_value, HDR_VALUE_3 "\t\t\t\t "); + TFW_STR(s3_value, HDR_VALUE_3); + TFW_STR(s3_rws, "\t\t\t\t "); const char *t_s3 = HDR_NAME_3 HDR_VALUE_3; unsigned long t_s3_len = strlen(t_s3); - collect_compound_str(s1, s1_lws, 0); + collect_compound_str(s1, col, 0); + collect_compound_str(s1, s1_lws, TFW_STR_OWS); collect_compound_str(s1, s1_value, 0); + collect_compound_str(s1, s1_rws, TFW_STR_OWS); + collect_compound_str(s2, col, 0); collect_compound_str(s2, s2_value, 0); - collect_compound_str(s3, s3_lws, 0); + collect_compound_str(s3, col, 0); + collect_compound_str(s3, s3_lws, TFW_STR_OWS); collect_compound_str(s3, s3_value, 0); + collect_compound_str(s3, s3_rws, TFW_STR_OWS); tbl = &ctx.hpack.enc_tbl; @@ -1470,21 +1521,27 @@ TEST(hpack, enc_table_rbtree) #define HDR_NAME_5 "test-foo-name" #define HDR_VALUE_5 "test-foo-value" - TFW_STR(s1, HDR_NAME_1 ":"); + TFW_STR(col, ":"); + TFW_STR(s1, HDR_NAME_1); TFW_STR(s1_value, HDR_VALUE_1); - TFW_STR(s2, HDR_NAME_2 ":"); + TFW_STR(s2, HDR_NAME_2); TFW_STR(s2_value, HDR_VALUE_2); - TFW_STR(s3, HDR_NAME_3 ":"); + TFW_STR(s3, HDR_NAME_3); TFW_STR(s3_value, HDR_VALUE_3); - TFW_STR(s4, HDR_NAME_4 ":"); + TFW_STR(s4, HDR_NAME_4); TFW_STR(s4_value, HDR_VALUE_4); - TFW_STR(s5, HDR_NAME_5 ":"); + TFW_STR(s5, HDR_NAME_5); TFW_STR(s5_value, HDR_VALUE_5); + collect_compound_str(s1, col, 0); collect_compound_str(s1, s1_value, 0); + collect_compound_str(s2, col, 0); collect_compound_str(s2, s2_value, 0); + collect_compound_str(s3, col, 0); collect_compound_str(s3, s3_value, 0); + collect_compound_str(s4, col, 0); collect_compound_str(s4, s4_value, 0); + collect_compound_str(s5, col, 0); collect_compound_str(s5, s5_value, 0); tbl = &ctx.hpack.enc_tbl; diff --git a/tempesta_fw/t/unit/test_http_parser.c b/tempesta_fw/t/unit/test_http_parser.c index de102cdac9..24b317c5c3 100644 --- a/tempesta_fw/t/unit/test_http_parser.c +++ b/tempesta_fw/t/unit/test_http_parser.c @@ -1763,7 +1763,8 @@ TEST(http_parser, cookie) unsigned int flags; const char *str; } kv[] = { - { 0, "Cookie: " }, + { 0, "Cookie:" }, + { TFW_STR_OWS, " " }, { TFW_STR_NAME, "session=" }, { TFW_STR_VALUE, "42" }, { 0, "; " }, @@ -1858,7 +1859,9 @@ TEST(http_parser, set_cookie) TfwStr *s_parsed = &resp->h_tbl->tbl[TFW_HTTP_HDR_SET_COOKIE]; TfwStr s_expected = { .chunks = (TfwStr []) { - { .data = "Set-Cookie: " , .len = 12 }, + { .data = "Set-Cookie:" , .len = 11 }, + { .data = " " , .len = 1, + .flags = TFW_STR_OWS }, { .data = "sessionid=" , .len = 10, .flags = TFW_STR_NAME }, { .data = "38afes7a8" , .len = 9, @@ -1866,7 +1869,7 @@ TEST(http_parser, set_cookie) { .data = "; HttpOnly; Path=/" , .len = 18 } }, .len = 49, - .nchunks = 4 + .nchunks = 5 }; test_string_split(&s_expected, s_parsed); } @@ -1881,7 +1884,9 @@ TEST(http_parser, set_cookie) TfwStr *s_parsed = &resp->h_tbl->tbl[TFW_HTTP_HDR_SET_COOKIE]; TfwStr s_expected = { .chunks = (TfwStr []) { - { .data = "Set-Cookie: " , .len = 12 }, + { .data = "Set-Cookie:" , .len = 11 }, + { .data = " " , .len = 1, + .flags = TFW_STR_OWS }, { .data = "sessionid=" , .len = 10, .flags = TFW_STR_NAME }, { .data = "\"38afes7a8\"" , .len = 11, @@ -1889,7 +1894,7 @@ TEST(http_parser, set_cookie) { .data = "; HttpOnly; Path=/" , .len = 18 } }, .len = 51, - .nchunks = 4 + .nchunks = 5 }; test_string_split(&s_expected, s_parsed); } @@ -1904,7 +1909,9 @@ TEST(http_parser, set_cookie) TfwStr *s_parsed = &resp->h_tbl->tbl[TFW_HTTP_HDR_SET_COOKIE]; TfwStr s_expected = { .chunks = (TfwStr []) { - { .data = "Set-Cookie: " , .len = 12 }, + { .data = "Set-Cookie:" , .len = 11 }, + { .data = " " , .len = 1, + .flags = TFW_STR_OWS }, { .data = "id=" , .len = 3, .flags = TFW_STR_NAME }, { .data = "a3fWa" , .len = 5, @@ -1914,7 +1921,7 @@ TEST(http_parser, set_cookie) .len = 57 } }, .len = 77, - .nchunks = 4 + .nchunks = 5 }; test_string_split(&s_expected, s_parsed); } @@ -1928,7 +1935,9 @@ TEST(http_parser, set_cookie) TfwStr *s_parsed = &resp->h_tbl->tbl[TFW_HTTP_HDR_SET_COOKIE]; TfwStr s_expected = { .chunks = (TfwStr []) { - { .data = "Set-Cookie: " , .len = 12 }, + { .data = "Set-Cookie:" , .len = 11 }, + { .data = " " , .len = 1, + .flags = TFW_STR_OWS }, { .data = "__Host-id=" , .len = 10, .flags = TFW_STR_NAME }, { .data = "1" , .len = 1, @@ -1937,7 +1946,7 @@ TEST(http_parser, set_cookie) .len = 36 } }, .len = 59, - .nchunks = 4 + .nchunks = 5 }; test_string_split(&s_expected, s_parsed); } diff --git a/tempesta_fw/tempesta_fw.h b/tempesta_fw/tempesta_fw.h index 2d30d8d3c2..81cab61cc4 100644 --- a/tempesta_fw/tempesta_fw.h +++ b/tempesta_fw/tempesta_fw.h @@ -34,6 +34,7 @@ #define TFW_AUTHOR "Tempesta Technologies, Inc" #define TFW_NAME "Tempesta FW" #define TFW_VERSION "0.7.0" +#define TFW_SERVER TFW_NAME "/" TFW_VERSION #define DEF_MAX_PORTS 8 diff --git a/tempesta_fw/vhost.c b/tempesta_fw/vhost.c index db7295ea8d..e9178e6d8d 100644 --- a/tempesta_fw/vhost.c +++ b/tempesta_fw/vhost.c @@ -333,7 +333,7 @@ tfw_vhost_get_hdr_mods(TfwLocation *loc, TfwVhost *vhost, int mod_type) loc = vhost->loc_dflt; if (!loc || !loc->mod_hdrs[mod_type].sz) loc = vh_dflt ? vh_dflt->loc_dflt : NULL; - if (!loc) + if (!loc || !loc->mod_hdrs[mod_type].sz) return NULL; return &loc->mod_hdrs[mod_type]; diff --git a/tempesta_fw/vhost.h b/tempesta_fw/vhost.h index 7dab3334c9..3ec2b3911a 100644 --- a/tempesta_fw/vhost.h +++ b/tempesta_fw/vhost.h @@ -68,11 +68,11 @@ typedef struct { * @hdr - Header string, see @tfw_http_msg_hdr_xfrm_str(); * @add_hdrs - Headers to modify; */ -typedef struct { +struct tfw_hdr_mods_desc_t { TfwStr *hdr; unsigned int hid; bool append; -} TfwHdrModsDesc; +}; /** * Headers modification before forwarding HTTP message. @@ -80,10 +80,10 @@ typedef struct { * @sz - Number of headers to modify; * @hdrs - Headers to modify; */ -typedef struct { +struct tfw_hdr_mods_t { size_t sz; TfwHdrModsDesc *hdrs; -} TfwHdrMods; +}; enum { TFW_VHOST_HDRMOD_REQ,