From 6426f9280339b132fd67c8a1a4cab396018d3925 Mon Sep 17 00:00:00 2001 From: matthiasL-scality Date: Fri, 17 Jul 2026 14:40:43 +0200 Subject: [PATCH 1/2] Fix GCS ':' encoding for proxied multipart complete and abort MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GCS normalises ':' to '%3A' in the canonical resource whenever S3 subresource params (uploadId, partNumber) are present — not only for presigned URLs but also for Authorization-header proxied requests. MULTIPART_COMPLETE and MULTIPART_ABORT both include '?uploadId=X' in the canonical resource, so GCS rejects them with 403 when the key contains literal ':'. MULTIPART_UPLOAD_PART has the same issue. For GCS backends (ENDPOINT_URL contains 'googleapis'), encode ':' as '%3A' in encoded_key before signing. The proxy_pass URL uses $encoded_key so the path sent to GCS is also updated automatically. Standard S3-compatible backends (cloudserver, Scaleway) keep literal ':'. MULTIPART_INITIATE uses '?uploads' (no value) and is unaffected. Co-Authored-By: Claude Sonnet 4.6 --- lua/compute_aws_s3_signature.lua | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lua/compute_aws_s3_signature.lua b/lua/compute_aws_s3_signature.lua index 4c6bfad..408bdf6 100644 --- a/lua/compute_aws_s3_signature.lua +++ b/lua/compute_aws_s3_signature.lua @@ -291,6 +291,12 @@ elseif signature_mode == "MULTIPART_UPLOAD_PART" then if not part_number or part_number == "" or not upload_id or upload_id == "" then return ngx.exit(ngx.HTTP_BAD_REQUEST) end + -- GCS normalises ':' to '%3A' in canonical resources when S3 subresource params + -- (partNumber, uploadId) are present. Apply to both the signature and the proxy URL + -- ($encoded_key). Standard backends keep literal ':'. + if (os.getenv('ENDPOINT_URL') or ''):find('googleapis', 1, true) then + ngx.var.encoded_key = ngx.var.encoded_key:gsub(':', '%%3A') + end compute_S3_signature_with_resource( "x-amz-date:" .. ngx.var.x_amz_date, "/" .. ngx.var.aws_tgt_bucket .. "/" .. ngx.var.encoded_key .. "?partNumber=" .. part_number .. "&uploadId=" .. upload_id @@ -309,6 +315,11 @@ elseif signature_mode == "MULTIPART_COMPLETE" then if not upload_id or upload_id == "" then return ngx.exit(ngx.HTTP_BAD_REQUEST) end + -- GCS normalises ':' to '%3A' in canonical resources when S3 subresource params + -- (uploadId) are present. Apply to both the signature and the proxy URL ($encoded_key). + if (os.getenv('ENDPOINT_URL') or ''):find('googleapis', 1, true) then + ngx.var.encoded_key = ngx.var.encoded_key:gsub(':', '%%3A') + end compute_S3_signature_with_resource( "x-amz-date:" .. ngx.var.x_amz_date, "/" .. ngx.var.aws_tgt_bucket .. "/" .. ngx.var.encoded_key .. "?uploadId=" .. upload_id @@ -327,6 +338,11 @@ elseif signature_mode == "MULTIPART_ABORT" then if not upload_id or upload_id == "" then return ngx.exit(ngx.HTTP_BAD_REQUEST) end + -- GCS normalises ':' to '%3A' in canonical resources when S3 subresource params + -- (uploadId) are present. Apply to both the signature and the proxy URL ($encoded_key). + if (os.getenv('ENDPOINT_URL') or ''):find('googleapis', 1, true) then + ngx.var.encoded_key = ngx.var.encoded_key:gsub(':', '%%3A') + end compute_S3_signature_with_resource( "x-amz-date:" .. ngx.var.x_amz_date, "/" .. ngx.var.aws_tgt_bucket .. "/" .. ngx.var.encoded_key .. "?uploadId=" .. upload_id From 73e8b55e57ff323e808c57ec150f6344c04944b5 Mon Sep 17 00:00:00 2001 From: matthiasL-scality Date: Fri, 17 Jul 2026 14:56:59 +0200 Subject: [PATCH 2/2] Fix: GCS excludes uploadId/partNumber from canonical resource MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The test showed that GCS rejects both literal ':' and '%3A' variants when '?uploadId=X' is included in the canonical resource. GCS computes: POST\n\napplication/xml\n\nx-amz-date:...\n/bucket/key%3A... No '?uploadId=X' — consistent with the presigned URL behaviour where '?partNumber=N&uploadId=X' is also excluded. Update MULTIPART_COMPLETE, MULTIPART_ABORT, and MULTIPART_UPLOAD_PART: for GCS, encode ':' as '%3A' AND omit uploadId/partNumber subresources from the canonical resource (the URL still carries them for GCS to route the operation; only the signature computation omits them). Standard S3-compatible backends keep literal ':' and include subresources. Co-Authored-By: Claude Sonnet 4.6 --- lua/compute_aws_s3_signature.lua | 55 +++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 19 deletions(-) diff --git a/lua/compute_aws_s3_signature.lua b/lua/compute_aws_s3_signature.lua index 408bdf6..27c6e8e 100644 --- a/lua/compute_aws_s3_signature.lua +++ b/lua/compute_aws_s3_signature.lua @@ -291,16 +291,21 @@ elseif signature_mode == "MULTIPART_UPLOAD_PART" then if not part_number or part_number == "" or not upload_id or upload_id == "" then return ngx.exit(ngx.HTTP_BAD_REQUEST) end - -- GCS normalises ':' to '%3A' in canonical resources when S3 subresource params - -- (partNumber, uploadId) are present. Apply to both the signature and the proxy URL - -- ($encoded_key). Standard backends keep literal ':'. + -- GCS does not include partNumber/uploadId in the canonical resource and + -- normalises ':' to '%3A' in the key — same quirks as for presigned part PUTs. + -- Standard S3-compatible backends include subresources and keep literal ':'. if (os.getenv('ENDPOINT_URL') or ''):find('googleapis', 1, true) then ngx.var.encoded_key = ngx.var.encoded_key:gsub(':', '%%3A') + compute_S3_signature_with_resource( + "x-amz-date:" .. ngx.var.x_amz_date, + "/" .. ngx.var.aws_tgt_bucket .. "/" .. ngx.var.encoded_key + ) + else + compute_S3_signature_with_resource( + "x-amz-date:" .. ngx.var.x_amz_date, + "/" .. ngx.var.aws_tgt_bucket .. "/" .. ngx.var.encoded_key .. "?partNumber=" .. part_number .. "&uploadId=" .. upload_id + ) end - compute_S3_signature_with_resource( - "x-amz-date:" .. ngx.var.x_amz_date, - "/" .. ngx.var.aws_tgt_bucket .. "/" .. ngx.var.encoded_key .. "?partNumber=" .. part_number .. "&uploadId=" .. upload_id - ) elseif signature_mode == "MULTIPART_COMPLETE" then @@ -315,15 +320,21 @@ elseif signature_mode == "MULTIPART_COMPLETE" then if not upload_id or upload_id == "" then return ngx.exit(ngx.HTTP_BAD_REQUEST) end - -- GCS normalises ':' to '%3A' in canonical resources when S3 subresource params - -- (uploadId) are present. Apply to both the signature and the proxy URL ($encoded_key). + -- GCS does not include uploadId in the canonical resource and normalises + -- ':' to '%3A' in the key. Standard S3-compatible backends include the + -- subresource and keep literal ':'. if (os.getenv('ENDPOINT_URL') or ''):find('googleapis', 1, true) then ngx.var.encoded_key = ngx.var.encoded_key:gsub(':', '%%3A') + compute_S3_signature_with_resource( + "x-amz-date:" .. ngx.var.x_amz_date, + "/" .. ngx.var.aws_tgt_bucket .. "/" .. ngx.var.encoded_key + ) + else + compute_S3_signature_with_resource( + "x-amz-date:" .. ngx.var.x_amz_date, + "/" .. ngx.var.aws_tgt_bucket .. "/" .. ngx.var.encoded_key .. "?uploadId=" .. upload_id + ) end - compute_S3_signature_with_resource( - "x-amz-date:" .. ngx.var.x_amz_date, - "/" .. ngx.var.aws_tgt_bucket .. "/" .. ngx.var.encoded_key .. "?uploadId=" .. upload_id - ) elseif signature_mode == "MULTIPART_ABORT" then @@ -338,15 +349,21 @@ elseif signature_mode == "MULTIPART_ABORT" then if not upload_id or upload_id == "" then return ngx.exit(ngx.HTTP_BAD_REQUEST) end - -- GCS normalises ':' to '%3A' in canonical resources when S3 subresource params - -- (uploadId) are present. Apply to both the signature and the proxy URL ($encoded_key). + -- GCS does not include uploadId in the canonical resource and normalises + -- ':' to '%3A' in the key. Standard S3-compatible backends include the + -- subresource and keep literal ':'. if (os.getenv('ENDPOINT_URL') or ''):find('googleapis', 1, true) then ngx.var.encoded_key = ngx.var.encoded_key:gsub(':', '%%3A') + compute_S3_signature_with_resource( + "x-amz-date:" .. ngx.var.x_amz_date, + "/" .. ngx.var.aws_tgt_bucket .. "/" .. ngx.var.encoded_key + ) + else + compute_S3_signature_with_resource( + "x-amz-date:" .. ngx.var.x_amz_date, + "/" .. ngx.var.aws_tgt_bucket .. "/" .. ngx.var.encoded_key .. "?uploadId=" .. upload_id + ) end - compute_S3_signature_with_resource( - "x-amz-date:" .. ngx.var.x_amz_date, - "/" .. ngx.var.aws_tgt_bucket .. "/" .. ngx.var.encoded_key .. "?uploadId=" .. upload_id - ) elseif signature_mode == "PRESIGN_PUT" then