From f531c55300b9daed8382a99700c35848ea36c277 Mon Sep 17 00:00:00 2001 From: devgs Date: Wed, 17 Nov 2021 11:30:27 +0200 Subject: [PATCH 1/3] Preserve some important headers in subrequests --- ngx_http_zip_headers.c | 23 +++++++++++++++++++++++ ngx_http_zip_module.c | 2 +- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/ngx_http_zip_headers.c b/ngx_http_zip_headers.c index 69f8039..a309b6f 100644 --- a/ngx_http_zip_headers.c +++ b/ngx_http_zip_headers.c @@ -209,6 +209,12 @@ ngx_int_t ngx_http_zip_init_subrequest_headers(ngx_http_request_t *r, ngx_http_request_t *sr, ngx_http_zip_range_t *piece_range, ngx_http_zip_range_t *req_range) { + ngx_list_t headers_old; + ngx_list_part_t *next_part; + ngx_table_elt_t *h; + ngx_uint_t i; + + ngx_memcpy(&headers_old, &sr->headers_in.headers, sizeof(headers_old)); ngx_memzero(&sr->headers_in, sizeof(sr->headers_in)); sr->headers_in.content_length_n = -1; sr->headers_in.keep_alive_n = -1; @@ -217,6 +223,23 @@ ngx_http_zip_init_subrequest_headers(ngx_http_request_t *r, ngx_http_request_t * return NGX_ERROR; } + next_part = &headers_old.part; + + for (; next_part; next_part = next_part->next) { + h = next_part->elts; + + for (i = 0; i < next_part->nelts; ++i) { + if ((!ngx_rstrncasecmp(h[i].key.data, "X-", sizeof("X-") - 1) + && ngx_rstrncasecmp(h[i].key.data, "X-Range", sizeof("X-Range") - 1)) + || !ngx_rstrncasecmp(h[i].key.data, "Authorization", sizeof("Authorization") - 1) + || !ngx_rstrncasecmp(h[i].key.data, "Cookie", sizeof("Cookie") - 1) + || !ngx_rstrncasecmp(h[i].key.data, "User-Agent", sizeof("User-Agent") - 1) + ) { + ngx_memcpy(ngx_list_push(&sr->headers_in.headers), &h[i], sizeof(ngx_table_elt_t)); + } + } + } + if (req_range && (piece_range->start < req_range->start || piece_range->end > req_range->end)) { ngx_table_elt_t *range_header = ngx_list_push(&sr->headers_in.headers); off_t start = req_range->start - piece_range->start; diff --git a/ngx_http_zip_module.c b/ngx_http_zip_module.c index 97e118c..b63dc1f 100644 --- a/ngx_http_zip_module.c +++ b/ngx_http_zip_module.c @@ -560,7 +560,7 @@ ngx_http_zip_send_file_piece(ngx_http_request_t *r, ngx_http_zip_ctx_t *ctx, return NGX_ERROR; } - if ((sr_ctx = ngx_pcalloc(r->pool, sizeof(ngx_http_zip_ctx_t))) == NULL) { + if ((sr_ctx = ngx_pcalloc(r->pool, sizeof(ngx_http_zip_sr_ctx_t))) == NULL) { return NGX_ERROR; } From d1d763b40683f96d3bbefd0f8af86a31ab772f4a Mon Sep 17 00:00:00 2001 From: devgs Date: Thu, 18 Nov 2021 10:31:02 +0200 Subject: [PATCH 2/3] Made the list of HTTP header fileds, that have to be included in sub-requests, configurable --- README.markdown | 6 ++++ ngx_http_zip_file.c | 40 +++++++++++++++++++++++++++ ngx_http_zip_headers.c | 62 ++++++++++++++++++++++++++---------------- ngx_http_zip_headers.h | 5 ++-- ngx_http_zip_module.c | 5 ++-- ngx_http_zip_module.h | 1 + 6 files changed, 92 insertions(+), 27 deletions(-) diff --git a/README.markdown b/README.markdown index bf91cc8..1719723 100644 --- a/README.markdown +++ b/README.markdown @@ -51,6 +51,12 @@ request returns any sort of error, the download is aborted. The CRC-32 is optional. Put "-" if you don't know the CRC-32; note that in this case mod_zip will disable support for the `Range` header. +If you want mod_zip to include some HTTP headers of the original request, in the +sub-requests that fetch content of files, then pass the list of their names in +the following HTTP header: + + X-Archive-Pass-Headers: [:]* + Re-encoding filenames --- diff --git a/ngx_http_zip_file.c b/ngx_http_zip_file.c index bf1bc47..c974dd3 100644 --- a/ngx_http_zip_file.c +++ b/ngx_http_zip_file.c @@ -12,6 +12,7 @@ static ngx_str_t ngx_http_zip_header_charset_name = ngx_string("upstream_http_x_archive_charset"); #endif static ngx_str_t ngx_http_zip_header_name_separator = ngx_string("upstream_http_x_archive_name_sep"); +static ngx_str_t ngx_http_zip_header_name_pass_headers = ngx_string("upstream_http_x_archive_pass_headers"); #define NGX_MAX_UINT16_VALUE 0xffff @@ -428,6 +429,45 @@ ngx_http_zip_generate_pieces(ngx_http_request_t *r, ngx_http_zip_ctx_t *ctx) ctx->archive_size = offset; + // Collect names of original request's header fields that + // have to be present in each of the issued sub-requests. + variable_header_status = ngx_http_variable_unknown_header(vv, &ngx_http_zip_header_name_pass_headers, + &r->upstream->headers_in.headers.part, sizeof("upstream_http_")-1); + + if (variable_header_status == NGX_OK && !vv->not_found) { + ngx_str_t *header; + ngx_int_t len; + u_char *next; + u_char *start = vv->data; + u_char *end = vv->data + vv->len; + + // Split the list of names by ':'. + while (start < end) { + next = ngx_strnstr(start, ":", end - start); + + if (next == NULL) { + next = end; + } + + len = next - start; + + if (len) { + if ((header = ngx_array_push(&ctx->pass_srq_headers)) == NULL) { + return NGX_ERROR; + } + + if ((header->data = ngx_pnalloc(r->pool, len)) == NULL) { + return NGX_ERROR; + } + + ngx_memcpy(header->data, start, len); + header->len = len; + } + + start = next + 1; + } + } + return NGX_OK; } diff --git a/ngx_http_zip_headers.c b/ngx_http_zip_headers.c index a309b6f..2586c5d 100644 --- a/ngx_http_zip_headers.c +++ b/ngx_http_zip_headers.c @@ -2,6 +2,22 @@ #include "ngx_http_zip_headers.h" #include "ngx_http_zip_file.h" +static ngx_uint_t +ngx_http_zip_find_key_in_set(ngx_str_t *key, ngx_array_t *set) +{ + ngx_uint_t i; + ngx_str_t *items = set->elts; + + for (i = 0; i < set->nelts; ++i) { + if (items[i].len == key->len + && !ngx_rstrncasecmp(items[i].data, key->data, key->len)) { + return 1; + } + } + + return 0; +} + ngx_int_t ngx_http_zip_add_cache_control(ngx_http_request_t *r) { @@ -206,40 +222,40 @@ ngx_http_zip_strip_range_header(ngx_http_request_t *r) } ngx_int_t -ngx_http_zip_init_subrequest_headers(ngx_http_request_t *r, ngx_http_request_t *sr, - ngx_http_zip_range_t *piece_range, ngx_http_zip_range_t *req_range) +ngx_http_zip_init_subrequest_headers(ngx_http_request_t *r, ngx_http_zip_ctx_t *ctx, + ngx_http_request_t *sr, ngx_http_zip_range_t *piece_range, + ngx_http_zip_range_t *req_range) { - ngx_list_t headers_old; - ngx_list_part_t *next_part; - ngx_table_elt_t *h; - ngx_uint_t i; + ngx_list_t new_headers; - ngx_memcpy(&headers_old, &sr->headers_in.headers, sizeof(headers_old)); - ngx_memzero(&sr->headers_in, sizeof(sr->headers_in)); - sr->headers_in.content_length_n = -1; - sr->headers_in.keep_alive_n = -1; - - if (ngx_list_init(&sr->headers_in.headers, r->pool, 1, sizeof(ngx_table_elt_t)) != NGX_OK) { + if (ngx_list_init(&new_headers, r->pool, 1, sizeof(ngx_table_elt_t)) != NGX_OK) { return NGX_ERROR; } - next_part = &headers_old.part; + if (ctx->pass_srq_headers.nelts) { + // Pass original header fileds, that appear on the list. + ngx_list_part_t *next_part; + ngx_table_elt_t *h; + ngx_uint_t i; + + next_part = &sr->headers_in.headers.part; - for (; next_part; next_part = next_part->next) { - h = next_part->elts; + for (; next_part; next_part = next_part->next) { + h = next_part->elts; - for (i = 0; i < next_part->nelts; ++i) { - if ((!ngx_rstrncasecmp(h[i].key.data, "X-", sizeof("X-") - 1) - && ngx_rstrncasecmp(h[i].key.data, "X-Range", sizeof("X-Range") - 1)) - || !ngx_rstrncasecmp(h[i].key.data, "Authorization", sizeof("Authorization") - 1) - || !ngx_rstrncasecmp(h[i].key.data, "Cookie", sizeof("Cookie") - 1) - || !ngx_rstrncasecmp(h[i].key.data, "User-Agent", sizeof("User-Agent") - 1) - ) { - ngx_memcpy(ngx_list_push(&sr->headers_in.headers), &h[i], sizeof(ngx_table_elt_t)); + for (i = 0; i < next_part->nelts; ++i) { + if (ngx_http_zip_find_key_in_set(&h[i].key, &ctx->pass_srq_headers)) { + ngx_memcpy(ngx_list_push(&new_headers), &h[i], sizeof(ngx_table_elt_t)); + } } } } + ngx_memzero(&sr->headers_in, sizeof(sr->headers_in)); + ngx_memcpy(&sr->headers_in.headers, &new_headers, sizeof(new_headers)); + sr->headers_in.content_length_n = -1; + sr->headers_in.keep_alive_n = -1; + if (req_range && (piece_range->start < req_range->start || piece_range->end > req_range->end)) { ngx_table_elt_t *range_header = ngx_list_push(&sr->headers_in.headers); off_t start = req_range->start - piece_range->start; diff --git a/ngx_http_zip_headers.h b/ngx_http_zip_headers.h index f183bfc..ab2900d 100644 --- a/ngx_http_zip_headers.h +++ b/ngx_http_zip_headers.h @@ -10,5 +10,6 @@ ngx_int_t ngx_http_zip_add_partial_content_range(ngx_http_request_t *r, ngx_int_t ngx_http_zip_init_multipart_range(ngx_http_request_t *r, ngx_http_zip_ctx_t *ctx); -ngx_int_t ngx_http_zip_init_subrequest_headers(ngx_http_request_t *r, ngx_http_request_t *sr, - ngx_http_zip_range_t *piece_range, ngx_http_zip_range_t *req_range); +ngx_int_t ngx_http_zip_init_subrequest_headers(ngx_http_request_t *r, ngx_http_zip_ctx_t *ctx, + ngx_http_request_t *sr, ngx_http_zip_range_t *piece_range, + ngx_http_zip_range_t *req_range); diff --git a/ngx_http_zip_module.c b/ngx_http_zip_module.c index b63dc1f..fb92dab 100644 --- a/ngx_http_zip_module.c +++ b/ngx_http_zip_module.c @@ -219,7 +219,8 @@ ngx_http_zip_main_request_header_filter(ngx_http_request_t *r) if ((ctx = ngx_pcalloc(r->pool, sizeof(ngx_http_zip_ctx_t))) == NULL || ngx_array_init(&ctx->files, r->pool, 1, sizeof(ngx_http_zip_file_t)) == NGX_ERROR - || ngx_array_init(&ctx->ranges, r->pool, 1, sizeof(ngx_http_zip_range_t)) == NGX_ERROR) + || ngx_array_init(&ctx->ranges, r->pool, 1, sizeof(ngx_http_zip_range_t)) == NGX_ERROR + || ngx_array_init(&ctx->pass_srq_headers, r->pool, 1, sizeof(ngx_str_t)) == NGX_ERROR) return NGX_ERROR; ngx_http_set_ctx(r, ctx, ngx_http_zip_module); @@ -550,7 +551,7 @@ ngx_http_zip_send_file_piece(ngx_http_request_t *r, ngx_http_zip_ctx_t *ctx, sr->subrequest_ranges = 1; sr->single_range = 1; - rc = ngx_http_zip_init_subrequest_headers(r, sr, &piece->range, req_range); + rc = ngx_http_zip_init_subrequest_headers(r, ctx, sr, &piece->range, req_range); if (sr->headers_in.range) { ngx_log_debug3(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "mod_zip: subrequest for \"%V?%V\" Range: %V", diff --git a/ngx_http_zip_module.h b/ngx_http_zip_module.h index 068b3a9..7a44de3 100644 --- a/ngx_http_zip_module.h +++ b/ngx_http_zip_module.h @@ -66,6 +66,7 @@ typedef struct { off_t archive_size; off_t cd_size; // zip central directory size ngx_http_request_t *wait; + ngx_array_t pass_srq_headers; unsigned parsed:1; unsigned trailer_sent:1; From 2fbec1d0ae0b809f533dfe894781efab56ecd4e6 Mon Sep 17 00:00:00 2001 From: devgs Date: Thu, 18 Nov 2021 17:32:18 +0200 Subject: [PATCH 3/3] Covered a sub-request header field passing feature with tests --- t/nginx.conf | 27 +++++++++++++-- t/nginx/html/zip-authorized-files-cookie.txt | 2 ++ t/nginx/html/zip-authorized-files-mixed.txt | 2 ++ t/nginx/html/zip-authorized-files-x.txt | 2 ++ t/ziptest.pl | 36 ++++++++++++++++++-- 5 files changed, 63 insertions(+), 6 deletions(-) create mode 100644 t/nginx/html/zip-authorized-files-cookie.txt create mode 100644 t/nginx/html/zip-authorized-files-mixed.txt create mode 100644 t/nginx/html/zip-authorized-files-x.txt diff --git a/t/nginx.conf b/t/nginx.conf index 13adfab..d5add33 100644 --- a/t/nginx.conf +++ b/t/nginx.conf @@ -47,10 +47,27 @@ http { } location /zip { - add_header X-Archive-Files zip; - add_header Last-Modified "Wed, 15 Nov 1995 04:58:08 GMT"; - add_header ETag "3.14159"; + add_header X-Archive-Files zip; + add_header X-Archive-Pass-Headers $arg_pass_headers; + add_header Last-Modified "Wed, 15 Nov 1995 04:58:08 GMT"; + add_header ETag "3.14159"; } + + location /with_auth/cookie { + if ($http_cookie = "") { + return 403; + } + + alias html; + } + + location /with_auth/x_auth_token { + if ($http_x_auth_token = "") { + return 403; + } + + alias html; + } } server { @@ -72,6 +89,10 @@ http { alias html; } + location /with_auth/ { + proxy_pass http://ziplist; + } + #error_page 404 /404.html; # redirect server error pages to the static page /50x.html diff --git a/t/nginx/html/zip-authorized-files-cookie.txt b/t/nginx/html/zip-authorized-files-cookie.txt new file mode 100644 index 0000000..b2e3481 --- /dev/null +++ b/t/nginx/html/zip-authorized-files-cookie.txt @@ -0,0 +1,2 @@ +1a6349c5 24 /with_auth/cookie/file1.txt file1.txt +5d70c4d3 25 /with_auth/cookie/file2.txt file2.txt diff --git a/t/nginx/html/zip-authorized-files-mixed.txt b/t/nginx/html/zip-authorized-files-mixed.txt new file mode 100644 index 0000000..8ffc0d3 --- /dev/null +++ b/t/nginx/html/zip-authorized-files-mixed.txt @@ -0,0 +1,2 @@ +1a6349c5 24 /with_auth/cookie/file1.txt file1.txt +5d70c4d3 25 /with_auth/x_auth_token/file2.txt file2.txt diff --git a/t/nginx/html/zip-authorized-files-x.txt b/t/nginx/html/zip-authorized-files-x.txt new file mode 100644 index 0000000..1fc30fc --- /dev/null +++ b/t/nginx/html/zip-authorized-files-x.txt @@ -0,0 +1,2 @@ +1a6349c5 24 /with_auth/x_auth_token/file1.txt file1.txt +5d70c4d3 25 /with_auth/x_auth_token/file2.txt file2.txt diff --git a/t/ziptest.pl b/t/ziptest.pl index 20d47cd..4fb25da 100755 --- a/t/ziptest.pl +++ b/t/ziptest.pl @@ -2,7 +2,7 @@ # TODO tests for Zip64 -use Test::More tests => 103; +use Test::More tests => 115; use LWP::UserAgent; use Archive::Zip; @@ -154,19 +154,49 @@ ($$) ########## Package empty directories $response = $ua->get("$http_root/zip-empty-dirs.txt"); -$zip = test_zip_archive($response->content, "with empty directories"); is($response->code, 200, "Returns OK with mixed empty directories and files"); +$zip = test_zip_archive($response->content, "with empty directories"); is($zip->memberNamed("file1.txt")->isBinaryFile(), 1, "file1.txt exists in archive"); is($zip->memberNamed("empty_dir1/")->isDirectory(), 1, "empty_dir1 exists in archive"); is($zip->memberNamed("file2.txt")->isBinaryFile(), 1, "file2.txt exists in archive"); is($zip->memberNamed("empty_dir2/")->isDirectory(), 1, "empty_dir2 exists in archive"); $response = $ua->get("$http_root/zip-only-empty-dirs.txt"); -$zip = write_temp_zip($response->content); is($response->code, 200, "Returns OK with only empty directories"); +$zip = write_temp_zip($response->content); is($zip->memberNamed("empty_dir1/")->isDirectory(), 1, "empty_dir1 exists in archive"); is($zip->memberNamed("empty_dir2/")->isDirectory(), 1, "empty_dir2 exists in archive"); +########## Pass headers in sub-requests + +$response = $ua->get("$http_root/zip-authorized-files-cookie.txt", "Cookie" => "session=verified"); +is($response->code, 500, "Server error on attempt to use authorizable file without Cookie"); + +$response = $ua->get("$http_root/zip-authorized-files-cookie.txt?pass_headers=Cookie", "Cookie" => "session=verified"); +is($response->code, 200, "Returns OK when Cookie header field is passed"); +$zip = write_temp_zip($response->content); +is($zip->memberNamed("file1.txt")->isBinaryFile(), 1, "file1.txt exists in archive"); +is($zip->memberNamed("file2.txt")->isBinaryFile(), 1, "file2.txt exists in archive"); + +$response = $ua->get("$http_root/zip-authorized-files-x.txt", "X-Auth-Token" => "verified"); +is($response->code, 500, "Server error on attempt to use authorizable file without X-Auth-Token"); + +$response = $ua->get("$http_root/zip-authorized-files-x.txt?pass_headers=X-Auth-Token", "X-Auth-Token" => "verified"); +is($response->code, 200, "Returns OK when X-Auth-Token header field is passed"); +$zip = write_temp_zip($response->content); +is($zip->memberNamed("file1.txt")->isBinaryFile(), 1, "file1.txt exists in archive"); +is($zip->memberNamed("file2.txt")->isBinaryFile(), 1, "file2.txt exists in archive"); + +$response = $ua->get("$http_root/zip-authorized-files-mixed.txt", "Cookie" => "session=verified", "X-Auth-Token" => "verified"); +is($response->code, 500, "Server error on attempt to use authorizable files without respective header fields"); + +$response = $ua->get("$http_root/zip-authorized-files-mixed.txt?pass_headers=Cookie:X-Auth-Token", + "Cookie" => "session=verified", "X-Auth-Token" => "verified"); +is($response->code, 200, "Returns OK when Cookie and X-Auth-Token header field is passed"); +$zip = write_temp_zip($response->content); +is($zip->memberNamed("file1.txt")->isBinaryFile(), 1, "file1.txt exists in archive"); +is($zip->memberNamed("file2.txt")->isBinaryFile(), 1, "file2.txt exists in archive"); + open LARGEFILE, ">", "nginx/html/largefile.txt"; for (0..99999) {