Skip to content

Commit

Permalink
Use Range header in subrequests
Browse files Browse the repository at this point in the history
  • Loading branch information
@evanmiller
evanmiller committed Oct 8, 2017
1 parent 67273e3 commit eb3f28c
Show file tree
Hide file tree
Showing 5 changed files with 94 additions and 121 deletions.
41 changes: 41 additions & 0 deletions ngx_http_zip_headers.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -205,3 +205,44 @@ ngx_http_zip_strip_range_header(ngx_http_request_t *r)
return NGX_OK;
}

ngx_int_t
ngx_http_zip_init_subrequest_headers(ngx_http_request_t *r, ngx_http_request_t *sr,
ngx_http_zip_range_t *piece_range, ngx_http_zip_range_t *req_range)
{
ngx_memzero(&sr->headers_in, sizeof(sr->headers_in));
sr->headers_in.content_length_n = -1;
sr->headers_in.keep_alive_n = -1;

if (ngx_list_init(&sr->headers_in.headers, r->pool, 1, sizeof(ngx_table_elt_t)) != NGX_OK) {
return NGX_ERROR;
}

if (req_range && (piece_range->start < req_range->start || piece_range->end > req_range->end)) {
ngx_table_elt_t *range_header = ngx_list_push(&sr->headers_in.headers);
off_t start = req_range->start - piece_range->start;
off_t end = req_range->end - piece_range->start;

if (start < 0)
start = 0;
if (end > piece_range->end)
end = piece_range->end;

if (range_header == NULL)
return NGX_ERROR;

range_header->value.data = ngx_pnalloc(r->pool, sizeof("bytes=-") + 2 * NGX_OFF_T_LEN);
if (range_header->value.data == NULL)
return NGX_ERROR;

range_header->value.len = ngx_sprintf(range_header->value.data, "bytes=%O-%O", start, end-1)
- range_header->value.data;
range_header->value.data[range_header->value.len] = '\0';

range_header->hash = 1;
ngx_str_set(&range_header->key, "Range");

sr->headers_in.range = range_header;
}

return NGX_OK;
}
2 changes: 2 additions & 0 deletions ngx_http_zip_headers.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,5 @@ ngx_int_t ngx_http_zip_add_partial_content_range(ngx_http_request_t *r,
ngx_int_t ngx_http_zip_init_multipart_range(ngx_http_request_t *r,
ngx_http_zip_ctx_t *ctx);

ngx_int_t ngx_http_zip_init_subrequest_headers(ngx_http_request_t *r, ngx_http_request_t *sr,
ngx_http_zip_range_t *piece_range, ngx_http_zip_range_t *req_range);
162 changes: 46 additions & 116 deletions ngx_http_zip_module.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,15 +41,15 @@ static ngx_int_t ngx_http_zip_subrequest_done(ngx_http_request_t *r, void *data,
static ngx_int_t ngx_http_zip_send_pieces(ngx_http_request_t *r,
ngx_http_zip_ctx_t *ctx);
static ngx_int_t ngx_http_zip_send_header_piece(ngx_http_request_t *r,
ngx_http_zip_ctx_t *ctx, ngx_http_zip_piece_t *piece, ngx_http_zip_range_t *range);
ngx_http_zip_ctx_t *ctx, ngx_http_zip_piece_t *piece, ngx_http_zip_range_t *req_range);
static ngx_int_t ngx_http_zip_send_file_piece(ngx_http_request_t *r,
ngx_http_zip_ctx_t *ctx, ngx_http_zip_piece_t *piece, ngx_http_zip_range_t *range);
ngx_http_zip_ctx_t *ctx, ngx_http_zip_piece_t *piece, ngx_http_zip_range_t *req_range);
static ngx_int_t ngx_http_zip_send_trailer_piece(ngx_http_request_t *r,
ngx_http_zip_ctx_t *ctx, ngx_http_zip_piece_t *piece, ngx_http_zip_range_t *range);
ngx_http_zip_ctx_t *ctx, ngx_http_zip_piece_t *piece, ngx_http_zip_range_t *req_range);
static ngx_int_t ngx_http_zip_send_central_directory_piece(ngx_http_request_t *r,
ngx_http_zip_ctx_t *ctx, ngx_http_zip_piece_t *piece, ngx_http_zip_range_t *range);
ngx_http_zip_ctx_t *ctx, ngx_http_zip_piece_t *piece, ngx_http_zip_range_t *req_range);
static ngx_int_t ngx_http_zip_send_piece(ngx_http_request_t *r,
ngx_http_zip_ctx_t *ctx, ngx_http_zip_piece_t *piece, ngx_http_zip_range_t *range);
ngx_http_zip_ctx_t *ctx, ngx_http_zip_piece_t *piece, ngx_http_zip_range_t *req_range);

static ngx_int_t ngx_http_zip_send_boundary(ngx_http_request_t *r,
ngx_http_zip_ctx_t *ctx, ngx_http_zip_range_t *range);
Expand Down Expand Up @@ -237,7 +237,8 @@ ngx_http_zip_subrequest_header_filter(ngx_http_request_t *r)

ctx = ngx_http_get_module_ctx(r->main, ngx_http_zip_module);
if (ctx != NULL) {
if (r->headers_out.status != NGX_HTTP_OK) {
if (r->headers_out.status != NGX_HTTP_OK &&
r->headers_out.status != NGX_HTTP_PARTIAL_CONTENT) {
ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
"mod_zip: a subrequest returned %d, aborting...",
r->headers_out.status);
Expand Down Expand Up @@ -360,24 +361,12 @@ ngx_http_zip_body_filter(ngx_http_request_t *r, ngx_chain_t *in)
static ngx_int_t
ngx_http_zip_subrequest_body_filter(ngx_http_request_t *r, ngx_chain_t *in)
{
ngx_http_zip_ctx_t *ctx;
ngx_http_zip_sr_ctx_t *sr_ctx;
ngx_chain_t *out;

ctx = ngx_http_get_module_ctx(r->main, ngx_http_zip_module);
sr_ctx = ngx_http_get_module_ctx(r, ngx_http_zip_module);

if (ctx && sr_ctx && in) {
if (sr_ctx->range != NULL) {
ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "mod_zip: cutting subrequest to fit range");
if ((out = ngx_http_zip_subrequest_range(r, in, sr_ctx)) == NULL)
return NGX_OK;

return ngx_http_next_body_filter(r, out);
}

ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "mod_zip: No range for subrequest to satisfy");

if (sr_ctx && in) {
if (sr_ctx->requesting_file->missing_crc32) {
ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "mod_zip: updating CRC-32");
ngx_http_zip_subrequest_update_crc32(in, sr_ctx->requesting_file);
Expand All @@ -387,80 +376,6 @@ ngx_http_zip_subrequest_body_filter(ngx_http_request_t *r, ngx_chain_t *in)
return ngx_http_next_body_filter(r, in);
}

/* more or less copied from ngx_http_range_filter_module.c */
static ngx_chain_t*
ngx_http_zip_subrequest_range(ngx_http_request_t *r, ngx_chain_t *in,
ngx_http_zip_sr_ctx_t *sr_ctx)
{
ngx_chain_t *out, *cl, **ll;
ngx_buf_t *buf;
ngx_http_zip_range_t *range;
off_t start, last;

range = sr_ctx->range;

out = NULL;
ll = &out;

for (cl = in; cl != NULL; cl = cl->next) {
buf = cl->buf;

start = sr_ctx->subrequest_pos;
last = sr_ctx->subrequest_pos + ngx_buf_size(buf);

sr_ctx->subrequest_pos = last;

ngx_log_debug4(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
"mod_zip: Buffer range %O-%O Request range %O-%O",
start, last, range->start, range->end);

if (range->end <= start || range->start >= last) {
ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
"mod_zip: range body skip");

if (buf->in_file) {
buf->file_pos = buf->file_last;
}

buf->pos = buf->last;
buf->sync = 1;

continue;
}

if (range->start > start) {
if (buf->in_file) {
buf->file_pos += range->start - start;
}

if (ngx_buf_in_memory(buf)) {
buf->pos += (size_t) (range->start - start);
}
}

if (range->end <= last) {
if (buf->in_file) {
buf->file_last -= last - range->end;
}

if (ngx_buf_in_memory(buf)) {
buf->last -= (size_t) (last - range->end);
}

buf->last_buf = 1;
*ll = cl;
cl->next = NULL;

break;
}

*ll = cl;
ll = &cl->next;
}

return out;
}

static ngx_int_t
ngx_http_zip_subrequest_update_crc32(ngx_chain_t *in,
ngx_http_zip_file_t *file)
Expand Down Expand Up @@ -588,7 +503,7 @@ ngx_http_zip_send_header_piece(ngx_http_request_t *r, ngx_http_zip_ctx_t *ctx,

static ngx_int_t
ngx_http_zip_send_file_piece(ngx_http_request_t *r, ngx_http_zip_ctx_t *ctx,
ngx_http_zip_piece_t *piece, ngx_http_zip_range_t *range)
ngx_http_zip_piece_t *piece, ngx_http_zip_range_t *req_range)
{
ngx_http_zip_sr_ctx_t *sr_ctx;
ngx_http_request_t *sr;
Expand Down Expand Up @@ -624,18 +539,33 @@ ngx_http_zip_send_file_piece(ngx_http_request_t *r, ngx_http_zip_ctx_t *ctx,

rc = ngx_http_subrequest(r, &piece->file->uri, &piece->file->args, &sr, ps, NGX_HTTP_SUBREQUEST_WAITED);
ngx_log_debug3(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
"mod_zip: subrequest for \"%V?%V\" initiated, result %d", &piece->file->uri, &piece->file->args, rc);
"mod_zip: subrequest for \"%V?%V\" initiated, result %d",
&piece->file->uri, &piece->file->args, rc);

if (rc == NGX_ERROR) {
return NGX_ERROR;
}

if ((sr_ctx = ngx_pcalloc(r->pool, sizeof(ngx_http_zip_ctx_t))) == NULL)
sr->allow_ranges = 1;
sr->subrequest_ranges = 1;
sr->single_range = 1;

rc = ngx_http_zip_init_subrequest_headers(r, sr, &piece->range, req_range);
if (sr->headers_in.range) {
ngx_log_debug3(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
"mod_zip: subrequest for \"%V?%V\" Range: %V",
&piece->file->uri, &piece->file->args, &sr->headers_in.range->value);
}
if (rc == NGX_ERROR) {
return NGX_ERROR;
}

if ((sr_ctx = ngx_pcalloc(r->pool, sizeof(ngx_http_zip_ctx_t))) == NULL) {
return NGX_ERROR;
}

sr_ctx->requesting_file = piece->file;
sr_ctx->subrequest_pos = piece->range.start;
sr_ctx->range = range;

ngx_http_set_ctx(sr, sr_ctx, ngx_http_zip_module);
if (ctx->wait) {
ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
Expand All @@ -648,14 +578,14 @@ ngx_http_zip_send_file_piece(ngx_http_request_t *r, ngx_http_zip_ctx_t *ctx,

static ngx_int_t
ngx_http_zip_send_trailer_piece(ngx_http_request_t *r, ngx_http_zip_ctx_t *ctx,
ngx_http_zip_piece_t *piece, ngx_http_zip_range_t *range)
ngx_http_zip_piece_t *piece, ngx_http_zip_range_t *req_range)
{
ngx_chain_t *link;

if (piece->file->missing_crc32) // should always be true, but if we somehow needed trailer piece - go on
ngx_crc32_final(piece->file->crc32);

if ((link = ngx_http_zip_data_descriptor_chain_link(r, piece, range)) == NULL) {
if ((link = ngx_http_zip_data_descriptor_chain_link(r, piece, req_range)) == NULL) {
ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "mod_zip: data descriptor failed");
return NGX_ERROR;
}
Expand All @@ -664,11 +594,11 @@ ngx_http_zip_send_trailer_piece(ngx_http_request_t *r, ngx_http_zip_ctx_t *ctx,

static ngx_int_t
ngx_http_zip_send_central_directory_piece(ngx_http_request_t *r, ngx_http_zip_ctx_t *ctx,
ngx_http_zip_piece_t *piece, ngx_http_zip_range_t *range)
ngx_http_zip_piece_t *piece, ngx_http_zip_range_t *req_range)
{
ngx_chain_t *link;

if ((link = ngx_http_zip_central_directory_chain_link(r, ctx, piece, range)) == NULL) {
if ((link = ngx_http_zip_central_directory_chain_link(r, ctx, piece, req_range)) == NULL) {
ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "mod_zip: CD piece failed");
return NGX_ERROR;
}
Expand All @@ -677,18 +607,18 @@ ngx_http_zip_send_central_directory_piece(ngx_http_request_t *r, ngx_http_zip_ct

static ngx_int_t
ngx_http_zip_send_piece(ngx_http_request_t *r, ngx_http_zip_ctx_t *ctx,
ngx_http_zip_piece_t *piece, ngx_http_zip_range_t *range)
ngx_http_zip_piece_t *piece, ngx_http_zip_range_t *req_range)
{
ngx_int_t rc = NGX_ERROR;

if (piece->type == zip_header_piece) {
rc = ngx_http_zip_send_header_piece(r, ctx, piece, range);
rc = ngx_http_zip_send_header_piece(r, ctx, piece, req_range);
} else if (piece->type == zip_file_piece) {
rc = ngx_http_zip_send_file_piece(r, ctx, piece, range);
rc = ngx_http_zip_send_file_piece(r, ctx, piece, req_range);
} else if (piece->type == zip_trailer_piece) {
rc = ngx_http_zip_send_trailer_piece(r, ctx, piece, range);
rc = ngx_http_zip_send_trailer_piece(r, ctx, piece, req_range);
} else if (piece->type == zip_central_directory_piece) {
rc = ngx_http_zip_send_central_directory_piece(r, ctx, piece, range);
rc = ngx_http_zip_send_central_directory_piece(r, ctx, piece, req_range);
}

return rc;
Expand Down Expand Up @@ -751,7 +681,7 @@ ngx_http_zip_send_pieces(ngx_http_request_t *r,
{
ngx_int_t rc = NGX_OK, pieces_sent = 0;
ngx_http_zip_piece_t *piece;
ngx_http_zip_range_t *range = NULL;
ngx_http_zip_range_t *req_range = NULL;

ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
"mod_zip: sending pieces, starting with piece %d of total %d", ctx->pieces_i, ctx->pieces_n);
Expand All @@ -766,31 +696,31 @@ ngx_http_zip_send_pieces(ngx_http_request_t *r,
}
break;
case 1:
range = &((ngx_http_zip_range_t *)ctx->ranges.elts)[0];
req_range = &((ngx_http_zip_range_t *)ctx->ranges.elts)[0];
while (rc == NGX_OK && ctx->pieces_i < ctx->pieces_n) {
piece = &ctx->pieces[ctx->pieces_i++];
if (ngx_http_zip_ranges_intersect(&piece->range, range)) {
if (ngx_http_zip_ranges_intersect(&piece->range, req_range)) {
pieces_sent++;
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "mod_zip: 1 range / sending piece type %d", piece->type);
rc = ngx_http_zip_send_piece(r, ctx, piece, range);
rc = ngx_http_zip_send_piece(r, ctx, piece, req_range);
}
}
break;
default:
while (rc == NGX_OK && ctx->ranges_i < ctx->ranges.nelts) {
range = &((ngx_http_zip_range_t *)ctx->ranges.elts)[ctx->ranges_i];
req_range = &((ngx_http_zip_range_t *)ctx->ranges.elts)[ctx->ranges_i];
ngx_log_debug4(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
"mod_zip: sending range #%d start=%O end=%O (size %d)",
ctx->ranges_i, range->start, range->end, range->boundary_header.len);
rc = ngx_http_zip_send_boundary(r, ctx, range);
ctx->ranges_i, req_range->start, req_range->end, req_range->boundary_header.len);
rc = ngx_http_zip_send_boundary(r, ctx, req_range);
while (rc == NGX_OK && ctx->pieces_i < ctx->pieces_n) {
piece = &ctx->pieces[ctx->pieces_i++];
if (ngx_http_zip_ranges_intersect(&piece->range, range)) {
if (ngx_http_zip_ranges_intersect(&piece->range, req_range)) {
ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
"mod_zip: sending range=%d piece=%d",
ctx->ranges_i, pieces_sent);
pieces_sent++;
rc = ngx_http_zip_send_piece(r, ctx, piece, range);
rc = ngx_http_zip_send_piece(r, ctx, piece, req_range);
}
}

Expand Down
2 changes: 0 additions & 2 deletions ngx_http_zip_module.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,5 @@ typedef struct {

typedef struct {
ngx_http_zip_file_t *requesting_file;
ngx_http_zip_range_t *range;
off_t subrequest_pos;
} ngx_http_zip_sr_ctx_t;

8 changes: 5 additions & 3 deletions t/ziptest.pl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,14 @@

sub set_debug_log($) {
my $label = shift;
my $debug = "debug";
$/ = "\n";
open( NEWCONF, ">", "nginx/conf/nginx.conf" );

open( CONF, "<", "nginx.conf" );
while(my $line = <CONF>) {
if ($line eq "error_log logs/error.log debug;\n") {
print NEWCONF "error_log logs/error-$label.log debug;\n";
print NEWCONF "error_log logs/error-$label.log $debug;\n";
} else {
print NEWCONF $line;
}
Expand Down Expand Up @@ -108,7 +109,7 @@ ($$)
$response = $ua->get("$http_root/zip-missing-crc.txt");
is($response->code, 200, "Returns OK with missing CRC");
like($response->header("Content-Length"), qr/^\d+$/, "Content-Length header when missing CRC");
is($response->header("Accept-Ranges"), undef, "No Accept-Ranges header when missing CRC (fails with nginx 0.7.44 - 0.8.6)");
is($response->header("Accept-Ranges"), undef, "No Accept-Ranges header when missing CRC");

$zip = test_zip_archive($response->content, "when missing CRC");
is($zip->memberNamed("file1.txt")->hasDataDescriptor(), 8, "Has data descriptor when missing CRC");
Expand Down Expand Up @@ -243,6 +244,7 @@ ($$)
is(substr($response->content, 0, 14), "the first file", "Subrange spanning part of first file");
is(substr($response->content, 68, 4), "This", "Subrange spanning part of second file");

# Subrange including part of first and second files (local).
$response = $ua->get("$http_root/zip-local-files.txt", "Range" => "bytes=".($file1_offset+9)."-".($file2_offset+4));
open TMPFILE, ">", "/tmp/partial.zip";
print TMPFILE $response->content;
Expand Down Expand Up @@ -313,4 +315,4 @@ ($$)
$response = $ua->get("$http_root/zip.txt",
"If-Range" => "3.14159",
"Range" => "bytes=0-1");
is($response->code, 206, "206 Partial Content -- when If-Range is ETag (requires nginx 0.8.10+ or nginx-0.8.9-etag.patch)");
is($response->code, 206, "206 Partial Content -- when If-Range is ETag");

0 comments on commit eb3f28c

Please sign in to comment.